2024-09-13T22:50:21.405 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-13T22:50:21.427 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T22:50:21.528 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904364 branch: squid description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/connectivity} email: ceph-qa@ceph.com first_in_suite: false job_id: '7904364' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_STRAY_DAEMON - CEPHADM_FAILED_DAEMON - CEPHADM_AGENT_DOWN log-only-match: - CEPHADM_ sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 workunit: branch: squid sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 3819 sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 sleep_before_teardown: 0 subset: 43/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 targets: smithi022.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKw3r0Nv3w/hfN71m0xZB/fmen4BziCRpuydRfBQmLoyds6WaJevALM23yGiX883SswwY9k17MdD5hy7qocIx9s= smithi076.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLFB+4HYKMLURdAzSUamuLzTH++xcZiDUEu+LVvoKK1tC7CUkPs0hiGX+H/UpZ49gkxaWQuVz+5L1aIBeJkiecM= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest false --force - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-13_21:08:03 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-13T22:50:21.528 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa; will attempt to use it 2024-09-13T22:50:21.529 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa/tasks 2024-09-13T22:50:21.529 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-13T22:50:21.530 INFO:teuthology.task.internal:Checking packages... 2024-09-13T22:50:21.551 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'ecab7be58963a5bd03910cd73eef66405d7fdd50' 2024-09-13T22:50:21.551 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T22:50:21.551 INFO:teuthology.packaging:ref: None 2024-09-13T22:50:21.552 INFO:teuthology.packaging:tag: None 2024-09-13T22:50:21.552 INFO:teuthology.packaging:branch: squid 2024-09-13T22:50:21.552 INFO:teuthology.packaging:sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:50:21.552 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-13T22:50:21.759 INFO:teuthology.task.internal:Found packages for ceph version 19.1.1-324.gecab7be5 2024-09-13T22:50:21.764 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-13T22:50:21.771 INFO:teuthology.task.internal:no buildpackages task found 2024-09-13T22:50:21.771 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-13T22:50:21.785 INFO:teuthology.task.internal:Saving configuration 2024-09-13T22:50:21.805 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-13T22:50:21.812 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-13T22:50:21.847 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi022.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904364', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 22:44:58.648244', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKw3r0Nv3w/hfN71m0xZB/fmen4BziCRpuydRfBQmLoyds6WaJevALM23yGiX883SswwY9k17MdD5hy7qocIx9s='} 2024-09-13T22:50:21.878 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi076.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904364', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 22:44:58.646656', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLFB+4HYKMLURdAzSUamuLzTH++xcZiDUEu+LVvoKK1tC7CUkPs0hiGX+H/UpZ49gkxaWQuVz+5L1aIBeJkiecM='} 2024-09-13T22:50:21.878 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-13T22:50:21.884 INFO:teuthology.task.internal:roles: ubuntu@smithi022.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-13T22:50:21.884 INFO:teuthology.task.internal:roles: ubuntu@smithi076.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-13T22:50:21.884 INFO:teuthology.run_tasks:Running task console_log... 2024-09-13T22:50:21.983 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fd14fe0bac0>, signals=[15]) 2024-09-13T22:50:21.983 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-13T22:50:21.989 INFO:teuthology.task.internal:Opening connections... 2024-09-13T22:50:21.989 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi022.front.sepia.ceph.com 2024-09-13T22:50:21.991 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:50:22.078 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi076.front.sepia.ceph.com 2024-09-13T22:50:22.079 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi076.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:50:22.182 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-13T22:50:22.189 DEBUG:teuthology.orchestra.run.smithi022:> uname -m 2024-09-13T22:50:22.208 INFO:teuthology.orchestra.run.smithi022.stdout:x86_64 2024-09-13T22:50:22.208 DEBUG:teuthology.orchestra.run.smithi022:> cat /etc/os-release 2024-09-13T22:50:22.269 INFO:teuthology.orchestra.run.smithi022.stdout:NAME="CentOS Stream" 2024-09-13T22:50:22.269 INFO:teuthology.orchestra.run.smithi022.stdout:VERSION="9" 2024-09-13T22:50:22.269 INFO:teuthology.orchestra.run.smithi022.stdout:ID="centos" 2024-09-13T22:50:22.269 INFO:teuthology.orchestra.run.smithi022.stdout:ID_LIKE="rhel fedora" 2024-09-13T22:50:22.269 INFO:teuthology.orchestra.run.smithi022.stdout:VERSION_ID="9" 2024-09-13T22:50:22.269 INFO:teuthology.orchestra.run.smithi022.stdout:PLATFORM_ID="platform:el9" 2024-09-13T22:50:22.270 INFO:teuthology.orchestra.run.smithi022.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T22:50:22.270 INFO:teuthology.orchestra.run.smithi022.stdout:ANSI_COLOR="0;31" 2024-09-13T22:50:22.270 INFO:teuthology.orchestra.run.smithi022.stdout:LOGO="fedora-logo-icon" 2024-09-13T22:50:22.270 INFO:teuthology.orchestra.run.smithi022.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T22:50:22.270 INFO:teuthology.orchestra.run.smithi022.stdout:HOME_URL="https://centos.org/" 2024-09-13T22:50:22.270 INFO:teuthology.orchestra.run.smithi022.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T22:50:22.270 INFO:teuthology.orchestra.run.smithi022.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T22:50:22.270 INFO:teuthology.orchestra.run.smithi022.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T22:50:22.271 INFO:teuthology.lock.ops:Updating smithi022.front.sepia.ceph.com on lock server 2024-09-13T22:50:22.307 DEBUG:teuthology.orchestra.run.smithi076:> uname -m 2024-09-13T22:50:22.326 INFO:teuthology.orchestra.run.smithi076.stdout:x86_64 2024-09-13T22:50:22.326 DEBUG:teuthology.orchestra.run.smithi076:> cat /etc/os-release 2024-09-13T22:50:22.384 INFO:teuthology.orchestra.run.smithi076.stdout:NAME="CentOS Stream" 2024-09-13T22:50:22.384 INFO:teuthology.orchestra.run.smithi076.stdout:VERSION="9" 2024-09-13T22:50:22.384 INFO:teuthology.orchestra.run.smithi076.stdout:ID="centos" 2024-09-13T22:50:22.384 INFO:teuthology.orchestra.run.smithi076.stdout:ID_LIKE="rhel fedora" 2024-09-13T22:50:22.384 INFO:teuthology.orchestra.run.smithi076.stdout:VERSION_ID="9" 2024-09-13T22:50:22.384 INFO:teuthology.orchestra.run.smithi076.stdout:PLATFORM_ID="platform:el9" 2024-09-13T22:50:22.384 INFO:teuthology.orchestra.run.smithi076.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T22:50:22.384 INFO:teuthology.orchestra.run.smithi076.stdout:ANSI_COLOR="0;31" 2024-09-13T22:50:22.384 INFO:teuthology.orchestra.run.smithi076.stdout:LOGO="fedora-logo-icon" 2024-09-13T22:50:22.385 INFO:teuthology.orchestra.run.smithi076.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T22:50:22.385 INFO:teuthology.orchestra.run.smithi076.stdout:HOME_URL="https://centos.org/" 2024-09-13T22:50:22.385 INFO:teuthology.orchestra.run.smithi076.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T22:50:22.385 INFO:teuthology.orchestra.run.smithi076.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T22:50:22.385 INFO:teuthology.orchestra.run.smithi076.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T22:50:22.385 INFO:teuthology.lock.ops:Updating smithi076.front.sepia.ceph.com on lock server 2024-09-13T22:50:22.416 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-13T22:50:22.424 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-13T22:50:22.446 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-13T22:50:22.446 DEBUG:teuthology.orchestra.run.smithi022:> test '!' -e /home/ubuntu/cephtest 2024-09-13T22:50:22.449 DEBUG:teuthology.orchestra.run.smithi076:> test '!' -e /home/ubuntu/cephtest 2024-09-13T22:50:22.466 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-13T22:50:22.472 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-13T22:50:22.472 DEBUG:teuthology.orchestra.run.smithi022:> test -z $(ls -A /var/lib/ceph) 2024-09-13T22:50:22.506 DEBUG:teuthology.orchestra.run.smithi076:> test -z $(ls -A /var/lib/ceph) 2024-09-13T22:50:22.605 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-13T22:50:22.677 INFO:teuthology.run_tasks:Running task kernel... 2024-09-13T22:50:22.692 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-13T22:50:22.692 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-13T22:50:22.692 DEBUG:teuthology.orchestra.run.smithi022:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T22:50:22.693 DEBUG:teuthology.orchestra.run.smithi076:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T22:50:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:50:22.710 DEBUG:teuthology.orchestra.run.smithi022:> uname -r 2024-09-13T22:50:22.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:50:22.711 DEBUG:teuthology.orchestra.run.smithi076:> uname -r 2024-09-13T22:50:22.768 INFO:teuthology.orchestra.run.smithi022.stdout:5.14.0-503.el9.x86_64 2024-09-13T22:50:22.768 INFO:teuthology.task.kernel:Running kernel on smithi022: 5.14.0-503.el9.x86_64 2024-09-13T22:50:22.768 DEBUG:teuthology.orchestra.run.smithi022:> sudo yum install -y kernel 2024-09-13T22:50:22.769 INFO:teuthology.orchestra.run.smithi076.stdout:5.14.0-503.el9.x86_64 2024-09-13T22:50:22.769 INFO:teuthology.task.kernel:Running kernel on smithi076: 5.14.0-503.el9.x86_64 2024-09-13T22:50:22.770 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum install -y kernel 2024-09-13T22:50:25.255 INFO:teuthology.orchestra.run.smithi076.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.2 MB 00:00 2024-09-13T22:50:25.379 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - BaseOS 9.1 MB/s | 8.2 MB 00:00 2024-09-13T22:50:27.213 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - AppStream 31 MB/s | 20 MB 00:00 2024-09-13T22:50:27.256 INFO:teuthology.orchestra.run.smithi076.stdout:CentOS Stream 9 - AppStream 24 MB/s | 20 MB 00:00 2024-09-13T22:50:34.322 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - CRB 2.1 MB/s | 6.5 MB 00:03 2024-09-13T22:50:34.997 INFO:teuthology.orchestra.run.smithi076.stdout:CentOS Stream 9 - CRB 1.7 MB/s | 6.5 MB 00:03 2024-09-13T22:50:35.756 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - Extras packages 100 kB/s | 19 kB 00:00 2024-09-13T22:50:36.640 INFO:teuthology.orchestra.run.smithi076.stdout:CentOS Stream 9 - Extras packages 90 kB/s | 19 kB 00:00 2024-09-13T22:50:36.693 INFO:teuthology.orchestra.run.smithi022.stdout:Extra Packages for Enterprise Linux 28 MB/s | 23 MB 00:00 2024-09-13T22:50:37.535 INFO:teuthology.orchestra.run.smithi076.stdout:Extra Packages for Enterprise Linux 29 MB/s | 23 MB 00:00 2024-09-13T22:50:42.366 INFO:teuthology.orchestra.run.smithi022.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-13T22:50:43.091 INFO:teuthology.orchestra.run.smithi076.stdout:lab-extras 14 kB/s | 1.7 kB 00:00 2024-09-13T22:50:44.481 INFO:teuthology.orchestra.run.smithi022.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T22:50:44.481 INFO:teuthology.orchestra.run.smithi022.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T22:50:44.577 INFO:teuthology.orchestra.run.smithi022.stdout:Dependencies resolved. 2024-09-13T22:50:44.588 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-09-13T22:50:44.588 INFO:teuthology.orchestra.run.smithi022.stdout: Package Architecture Version Repository Size 2024-09-13T22:50:44.588 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-09-13T22:50:44.588 INFO:teuthology.orchestra.run.smithi022.stdout:Installing: 2024-09-13T22:50:44.588 INFO:teuthology.orchestra.run.smithi022.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T22:50:44.588 INFO:teuthology.orchestra.run.smithi022.stdout:Installing dependencies: 2024-09-13T22:50:44.589 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T22:50:44.589 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T22:50:44.589 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T22:50:44.589 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T22:50:44.589 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction Summary 2024-09-13T22:50:44.589 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-09-13T22:50:44.589 INFO:teuthology.orchestra.run.smithi022.stdout:Install 4 Packages 2024-09-13T22:50:44.589 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T22:50:44.590 INFO:teuthology.orchestra.run.smithi022.stdout:Total download size: 79 M 2024-09-13T22:50:44.590 INFO:teuthology.orchestra.run.smithi022.stdout:Installed size: 126 M 2024-09-13T22:50:44.590 INFO:teuthology.orchestra.run.smithi022.stdout:Downloading Packages: 2024-09-13T22:50:44.934 INFO:teuthology.orchestra.run.smithi022.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 250 kB/s | 23 kB 00:00 2024-09-13T22:50:45.097 INFO:teuthology.orchestra.run.smithi076.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T22:50:45.097 INFO:teuthology.orchestra.run.smithi076.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T22:50:45.189 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-13T22:50:45.197 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-13T22:50:45.197 INFO:teuthology.orchestra.run.smithi076.stdout: Package Architecture Version Repository Size 2024-09-13T22:50:45.197 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-13T22:50:45.197 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-13T22:50:45.197 INFO:teuthology.orchestra.run.smithi076.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T22:50:45.197 INFO:teuthology.orchestra.run.smithi076.stdout:Installing dependencies: 2024-09-13T22:50:45.197 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T22:50:45.198 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T22:50:45.198 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T22:50:45.198 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T22:50:45.198 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-13T22:50:45.198 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-13T22:50:45.198 INFO:teuthology.orchestra.run.smithi076.stdout:Install 4 Packages 2024-09-13T22:50:45.198 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T22:50:45.199 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 79 M 2024-09-13T22:50:45.199 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 126 M 2024-09-13T22:50:45.199 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-13T22:50:45.317 INFO:teuthology.orchestra.run.smithi022.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 33 MB/s | 16 MB 00:00 2024-09-13T22:50:45.801 INFO:teuthology.orchestra.run.smithi022.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 33 MB/s | 29 MB 00:00 2024-09-13T22:50:45.984 INFO:teuthology.orchestra.run.smithi076.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 43 kB/s | 23 kB 00:00 2024-09-13T22:50:46.110 INFO:teuthology.orchestra.run.smithi022.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 27 MB/s | 35 MB 00:01 2024-09-13T22:50:46.110 INFO:teuthology.orchestra.run.smithi022.stdout:-------------------------------------------------------------------------------- 2024-09-13T22:50:46.110 INFO:teuthology.orchestra.run.smithi022.stdout:Total 52 MB/s | 79 MB 00:01 2024-09-13T22:50:46.823 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction check 2024-09-13T22:50:46.968 INFO:teuthology.orchestra.run.smithi076.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 10 MB/s | 16 MB 00:01 2024-09-13T22:50:47.239 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction check succeeded. 2024-09-13T22:50:47.239 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction test 2024-09-13T22:50:47.543 INFO:teuthology.orchestra.run.smithi076.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 17 MB/s | 35 MB 00:02 2024-09-13T22:50:48.177 INFO:teuthology.orchestra.run.smithi076.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 13 MB/s | 29 MB 00:02 2024-09-13T22:50:48.177 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-13T22:50:48.177 INFO:teuthology.orchestra.run.smithi076.stdout:Total 26 MB/s | 79 MB 00:02 2024-09-13T22:50:48.213 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction test succeeded. 2024-09-13T22:50:48.214 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction 2024-09-13T22:50:48.861 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-13T22:50:49.208 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-13T22:50:49.209 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-13T22:50:50.167 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-13T22:50:50.167 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-13T22:50:50.792 INFO:teuthology.orchestra.run.smithi022.stdout: Preparing : 1/1 2024-09-13T22:50:51.453 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T22:50:51.535 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T22:50:52.637 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-13T22:50:53.291 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T22:50:53.375 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T22:50:54.330 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T22:50:54.465 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T22:50:56.187 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T22:50:56.325 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T22:50:59.586 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T22:50:59.631 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:51:01.140 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T22:51:01.243 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:51:04.767 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:51:06.064 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:51:43.973 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:51:44.013 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:51:44.315 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:51:44.315 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T22:51:44.315 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T22:51:44.316 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T22:51:45.125 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:51:45.166 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:51:45.166 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T22:51:45.166 INFO:teuthology.orchestra.run.smithi022.stdout:Installed: 2024-09-13T22:51:45.166 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T22:51:45.166 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T22:51:45.166 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T22:51:45.167 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T22:51:45.167 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T22:51:45.167 INFO:teuthology.orchestra.run.smithi022.stdout:Complete! 2024-09-13T22:51:45.169 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:51:45.530 DEBUG:teuthology.orchestra.run.smithi022:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T22:51:45.550 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:51:45.550 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T22:51:45.551 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T22:51:45.551 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T22:51:45.619 INFO:teuthology.orchestra.run.smithi022.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T22:51:45.619 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T22:51:45.619 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T22:51:45.619 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T22:51:45.620 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T22:51:45.620 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-13T22:51:45.620 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi022.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T22:51:45.620 DEBUG:teuthology.orchestra.run.smithi022:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T22:51:45.678 INFO:teuthology.orchestra.run.smithi022.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T22:51:45.678 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T22:51:45.678 DEBUG:teuthology.orchestra.run.smithi022:> sudo rpm -qi grub2-tools 2024-09-13T22:51:45.716 INFO:teuthology.orchestra.run.smithi022.stdout:Name : grub2-tools 2024-09-13T22:51:45.717 INFO:teuthology.orchestra.run.smithi022.stdout:Epoch : 1 2024-09-13T22:51:45.717 INFO:teuthology.orchestra.run.smithi022.stdout:Version : 2.06 2024-09-13T22:51:45.717 INFO:teuthology.orchestra.run.smithi022.stdout:Release : 82.el9 2024-09-13T22:51:45.717 INFO:teuthology.orchestra.run.smithi022.stdout:Architecture: x86_64 2024-09-13T22:51:45.717 INFO:teuthology.orchestra.run.smithi022.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T22:51:45.717 INFO:teuthology.orchestra.run.smithi022.stdout:Group : Unspecified 2024-09-13T22:51:45.717 INFO:teuthology.orchestra.run.smithi022.stdout:Size : 8274670 2024-09-13T22:51:45.717 INFO:teuthology.orchestra.run.smithi022.stdout:License : GPLv3+ 2024-09-13T22:51:45.717 INFO:teuthology.orchestra.run.smithi022.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T22:51:45.717 INFO:teuthology.orchestra.run.smithi022.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T22:51:45.717 INFO:teuthology.orchestra.run.smithi022.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout:Packager : builder@centos.org 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout:Vendor : CentOS 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout:Summary : Support tools for GRUB. 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout:Description : 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout:hardware devices. 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T22:51:45.718 INFO:teuthology.orchestra.run.smithi022.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T22:51:45.719 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T22:51:45.719 INFO:teuthology.task.kernel:Updating grub on smithi022 to boot 5.14.0-505.el9.x86_64 2024-09-13T22:51:45.720 DEBUG:teuthology.orchestra.run.smithi022:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T22:51:46.359 INFO:teuthology.orchestra.run.smithi022.stderr:Generating grub configuration file ... 2024-09-13T22:51:46.452 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:51:46.452 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T22:51:46.452 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-13T22:51:46.452 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T22:51:46.452 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T22:51:46.452 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T22:51:46.452 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T22:51:46.452 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T22:51:46.452 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-13T22:51:46.807 DEBUG:teuthology.orchestra.run.smithi076:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T22:51:46.889 INFO:teuthology.orchestra.run.smithi076.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T22:51:46.890 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T22:51:46.890 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T22:51:46.890 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T22:51:46.890 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T22:51:46.890 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-13T22:51:46.890 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi076.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T22:51:46.890 DEBUG:teuthology.orchestra.run.smithi076:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T22:51:46.949 INFO:teuthology.orchestra.run.smithi076.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T22:51:46.949 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T22:51:46.949 DEBUG:teuthology.orchestra.run.smithi076:> sudo rpm -qi grub2-tools 2024-09-13T22:51:46.982 INFO:teuthology.orchestra.run.smithi076.stdout:Name : grub2-tools 2024-09-13T22:51:46.982 INFO:teuthology.orchestra.run.smithi076.stdout:Epoch : 1 2024-09-13T22:51:46.982 INFO:teuthology.orchestra.run.smithi076.stdout:Version : 2.06 2024-09-13T22:51:46.983 INFO:teuthology.orchestra.run.smithi076.stdout:Release : 82.el9 2024-09-13T22:51:46.983 INFO:teuthology.orchestra.run.smithi076.stdout:Architecture: x86_64 2024-09-13T22:51:46.983 INFO:teuthology.orchestra.run.smithi076.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T22:51:46.983 INFO:teuthology.orchestra.run.smithi076.stdout:Group : Unspecified 2024-09-13T22:51:46.983 INFO:teuthology.orchestra.run.smithi076.stdout:Size : 8274670 2024-09-13T22:51:46.983 INFO:teuthology.orchestra.run.smithi076.stdout:License : GPLv3+ 2024-09-13T22:51:46.983 INFO:teuthology.orchestra.run.smithi076.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T22:51:46.983 INFO:teuthology.orchestra.run.smithi076.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T22:51:46.983 INFO:teuthology.orchestra.run.smithi076.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T22:51:46.983 INFO:teuthology.orchestra.run.smithi076.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T22:51:46.983 INFO:teuthology.orchestra.run.smithi076.stdout:Packager : builder@centos.org 2024-09-13T22:51:46.984 INFO:teuthology.orchestra.run.smithi076.stdout:Vendor : CentOS 2024-09-13T22:51:46.984 INFO:teuthology.orchestra.run.smithi076.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T22:51:46.984 INFO:teuthology.orchestra.run.smithi076.stdout:Summary : Support tools for GRUB. 2024-09-13T22:51:46.984 INFO:teuthology.orchestra.run.smithi076.stdout:Description : 2024-09-13T22:51:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T22:51:46.984 INFO:teuthology.orchestra.run.smithi076.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T22:51:46.984 INFO:teuthology.orchestra.run.smithi076.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T22:51:46.984 INFO:teuthology.orchestra.run.smithi076.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T22:51:46.984 INFO:teuthology.orchestra.run.smithi076.stdout:hardware devices. 2024-09-13T22:51:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T22:51:46.984 INFO:teuthology.orchestra.run.smithi076.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T22:51:46.985 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T22:51:46.985 INFO:teuthology.task.kernel:Updating grub on smithi076 to boot 5.14.0-505.el9.x86_64 2024-09-13T22:51:46.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T22:51:47.658 INFO:teuthology.orchestra.run.smithi022.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T22:51:47.693 INFO:teuthology.orchestra.run.smithi022.stderr:done 2024-09-13T22:51:47.694 DEBUG:teuthology.orchestra.run.smithi022:> mktemp 2024-09-13T22:51:47.709 INFO:teuthology.orchestra.run.smithi022.stdout:/tmp/tmp.hlOnhz78iY 2024-09-13T22:51:47.709 DEBUG:teuthology.orchestra.run.smithi022:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.hlOnhz78iY 2024-09-13T22:51:47.744 INFO:teuthology.orchestra.run.smithi076.stderr:Generating grub configuration file ... 2024-09-13T22:51:47.776 DEBUG:teuthology.orchestra.run.smithi022:> sudo chmod 0666 /tmp/tmp.hlOnhz78iY 2024-09-13T22:51:47.911 DEBUG:teuthology.orchestra.remote:smithi022:/tmp/tmp.hlOnhz78iY is 6KB 2024-09-13T22:51:47.959 DEBUG:teuthology.orchestra.run.smithi022:> rm -fr /tmp/tmp.hlOnhz78iY 2024-09-13T22:51:47.973 DEBUG:teuthology.orchestra.run.smithi022:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T22:51:48.035 INFO:teuthology.orchestra.run.smithi022.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T22:51:48.035 INFO:teuthology.orchestra.run.smithi022.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T22:51:48.035 INFO:teuthology.orchestra.run.smithi022.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T22:51:48.036 INFO:teuthology.orchestra.run.smithi022.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T22:51:48.037 DEBUG:teuthology.orchestra.run.smithi022:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T22:51:48.188 DEBUG:teuthology.orchestra.run.smithi022:> sudo shutdown -r now 2024-09-13T22:51:49.077 INFO:teuthology.orchestra.run.smithi076.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T22:51:49.103 INFO:teuthology.orchestra.run.smithi076.stderr:done 2024-09-13T22:51:49.106 DEBUG:teuthology.orchestra.run.smithi076:> mktemp 2024-09-13T22:51:49.162 INFO:teuthology.orchestra.run.smithi076.stdout:/tmp/tmp.fKZFHqGglw 2024-09-13T22:51:49.162 DEBUG:teuthology.orchestra.run.smithi076:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.fKZFHqGglw 2024-09-13T22:51:49.229 DEBUG:teuthology.orchestra.run.smithi076:> sudo chmod 0666 /tmp/tmp.fKZFHqGglw 2024-09-13T22:51:49.370 DEBUG:teuthology.orchestra.remote:smithi076:/tmp/tmp.fKZFHqGglw is 6KB 2024-09-13T22:51:49.419 DEBUG:teuthology.orchestra.run.smithi076:> rm -fr /tmp/tmp.fKZFHqGglw 2024-09-13T22:51:49.434 DEBUG:teuthology.orchestra.run.smithi076:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T22:51:49.498 INFO:teuthology.orchestra.run.smithi076.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T22:51:49.498 INFO:teuthology.orchestra.run.smithi076.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T22:51:49.498 INFO:teuthology.orchestra.run.smithi076.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T22:51:49.498 INFO:teuthology.orchestra.run.smithi076.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T22:51:49.499 DEBUG:teuthology.orchestra.run.smithi076:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T22:51:49.649 DEBUG:teuthology.orchestra.run.smithi076:> sudo shutdown -r now 2024-09-13T22:52:18.221 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-13T22:52:18.222 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-09-13T22:52:18.223 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:52:19.653 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-13T22:52:19.654 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-13T22:52:19.654 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi076.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:52:36.801 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.22 2024-09-13T22:52:38.081 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.76 2024-09-13T22:52:45.810 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-09-13T22:52:45.811 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:52:47.083 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-13T22:52:47.084 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi076.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:52:50.177 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.76 2024-09-13T22:53:02.189 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-13T22:53:02.189 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi076.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:53:05.281 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.76 2024-09-13T22:53:20.296 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-13T22:53:20.297 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi076.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:53:45.845 DEBUG:teuthology.orchestra.remote:timed out 2024-09-13T22:53:52.210 DEBUG:teuthology.orchestra.run.smithi076:> true 2024-09-13T22:53:52.583 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-13T22:53:52.584 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-505.el9.x86_64"... 2024-09-13T22:53:52.584 DEBUG:teuthology.orchestra.run.smithi076:> uname -r 2024-09-13T22:53:52.601 INFO:teuthology.orchestra.run.smithi076.stdout:5.14.0-505.el9.x86_64 2024-09-13T22:53:52.601 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T22:53:52.602 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T22:53:52.602 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T22:53:53.602 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-13T22:53:53.602 DEBUG:teuthology.orchestra.run.smithi076:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T22:53:53.676 INFO:teuthology.orchestra.run.smithi076.stdout:ttyS1 2024-09-13T22:53:53.700 DEBUG:teuthology.parallel:result is None 2024-09-13T22:53:57.846 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-09-13T22:53:57.847 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:53:58.215 DEBUG:teuthology.orchestra.run.smithi022:> true 2024-09-13T22:53:58.571 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-09-13T22:53:58.571 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-13T22:53:58.571 DEBUG:teuthology.orchestra.run.smithi022:> uname -r 2024-09-13T22:53:58.587 INFO:teuthology.orchestra.run.smithi022.stdout:5.14.0-505.el9.x86_64 2024-09-13T22:53:58.588 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T22:53:58.588 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T22:53:58.588 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T22:53:59.588 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-13T22:53:59.589 DEBUG:teuthology.orchestra.run.smithi022:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T22:53:59.657 INFO:teuthology.orchestra.run.smithi022.stdout:ttyS1 2024-09-13T22:53:59.677 DEBUG:teuthology.parallel:result is None 2024-09-13T22:53:59.677 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-13T22:53:59.684 INFO:teuthology.task.internal:Creating test directory... 2024-09-13T22:53:59.685 DEBUG:teuthology.orchestra.run.smithi022:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T22:53:59.688 DEBUG:teuthology.orchestra.run.smithi076:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T22:53:59.707 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-13T22:53:59.716 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-13T22:53:59.724 INFO:teuthology.task.internal:Creating archive directory... 2024-09-13T22:53:59.725 DEBUG:teuthology.orchestra.run.smithi022:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T22:53:59.748 DEBUG:teuthology.orchestra.run.smithi076:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T22:53:59.847 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-13T22:53:59.855 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-13T22:53:59.855 DEBUG:teuthology.orchestra.run.smithi022:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T22:53:59.858 DEBUG:teuthology.orchestra.run.smithi076:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T22:53:59.888 INFO:teuthology.orchestra.run.smithi022.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T22:53:59.893 INFO:teuthology.orchestra.run.smithi076.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T22:53:59.900 INFO:teuthology.orchestra.run.smithi022.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T22:53:59.904 INFO:teuthology.orchestra.run.smithi076.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T22:53:59.906 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-13T22:53:59.912 INFO:teuthology.task.internal:Configuring sudo... 2024-09-13T22:53:59.913 DEBUG:teuthology.orchestra.run.smithi022:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T22:53:59.945 DEBUG:teuthology.orchestra.run.smithi076:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T22:53:59.973 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-13T22:53:59.983 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-13T22:53:59.983 DEBUG:teuthology.orchestra.run.smithi022:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T22:54:00.014 DEBUG:teuthology.orchestra.run.smithi076:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T22:54:00.031 DEBUG:teuthology.orchestra.run.smithi022:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T22:54:00.120 DEBUG:teuthology.orchestra.run.smithi022:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T22:54:00.159 DEBUG:teuthology.orchestra.run.smithi022:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T22:54:00.251 DEBUG:teuthology.orchestra.run.smithi022:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T22:54:00.285 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-13T22:54:00.285 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T22:54:00.352 DEBUG:teuthology.orchestra.run.smithi076:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T22:54:00.405 DEBUG:teuthology.orchestra.run.smithi076:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T22:54:00.451 DEBUG:teuthology.orchestra.run.smithi076:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T22:54:00.543 DEBUG:teuthology.orchestra.run.smithi076:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T22:54:00.575 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-13T22:54:00.575 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T22:54:00.641 DEBUG:teuthology.orchestra.run.smithi022:> sudo service rsyslog restart 2024-09-13T22:54:00.643 DEBUG:teuthology.orchestra.run.smithi076:> sudo service rsyslog restart 2024-09-13T22:54:00.689 INFO:teuthology.orchestra.run.smithi022.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T22:54:00.724 INFO:teuthology.orchestra.run.smithi076.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T22:54:01.235 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-13T22:54:01.244 INFO:teuthology.task.internal:Starting timer... 2024-09-13T22:54:01.244 INFO:teuthology.run_tasks:Running task pcp... 2024-09-13T22:54:01.254 INFO:teuthology.run_tasks:Running task selinux... 2024-09-13T22:54:01.263 DEBUG:teuthology.orchestra.run.smithi022:> sudo service auditd rotate 2024-09-13T22:54:01.329 INFO:teuthology.orchestra.run.smithi022.stdout:Rotating logs: 2024-09-13T22:54:01.331 DEBUG:teuthology.orchestra.run.smithi076:> sudo service auditd rotate 2024-09-13T22:54:01.391 INFO:teuthology.orchestra.run.smithi076.stdout:Rotating logs: 2024-09-13T22:54:01.393 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-13T22:54:01.393 DEBUG:teuthology.orchestra.run.smithi022:> /usr/sbin/getenforce 2024-09-13T22:54:01.422 INFO:teuthology.orchestra.run.smithi022.stdout:Permissive 2024-09-13T22:54:01.423 DEBUG:teuthology.orchestra.run.smithi076:> /usr/sbin/getenforce 2024-09-13T22:54:01.462 INFO:teuthology.orchestra.run.smithi076.stdout:Permissive 2024-09-13T22:54:01.462 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi022.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi076.front.sepia.ceph.com': 'permissive'} 2024-09-13T22:54:01.462 DEBUG:teuthology.orchestra.run.smithi022:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-13T22:54:01.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:54:01.493 DEBUG:teuthology.orchestra.run.smithi076:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-13T22:54:01.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:54:01.529 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-13T22:54:01.529 DEBUG:teuthology.orchestra.run.smithi022:> sudo /usr/sbin/setenforce permissive 2024-09-13T22:54:01.569 DEBUG:teuthology.orchestra.run.smithi076:> sudo /usr/sbin/setenforce permissive 2024-09-13T22:54:01.613 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-13T22:54:01.623 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-13T22:54:01.638 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-13T22:54:01.639 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi022.front.sepia.ceph.com,smithi076.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-13T22:57:51.245 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi022.front.sepia.ceph.com'), Remote(name='ubuntu@smithi076.front.sepia.ceph.com')] 2024-09-13T22:57:51.246 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-09-13T22:57:51.247 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:57:51.352 DEBUG:teuthology.orchestra.run.smithi022:> true 2024-09-13T22:57:51.442 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-09-13T22:57:51.443 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-13T22:57:51.443 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi076.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:57:51.538 DEBUG:teuthology.orchestra.run.smithi076:> true 2024-09-13T22:57:51.622 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-13T22:57:51.623 INFO:teuthology.run_tasks:Running task clock... 2024-09-13T22:57:51.635 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-13T22:57:51.636 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T22:57:51.637 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T22:57:51.639 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T22:57:51.640 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T22:57:51.671 INFO:teuthology.orchestra.run.smithi022.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T22:57:51.688 INFO:teuthology.orchestra.run.smithi022.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T22:57:51.695 INFO:teuthology.orchestra.run.smithi076.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T22:57:51.711 INFO:teuthology.orchestra.run.smithi076.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T22:57:51.716 INFO:teuthology.orchestra.run.smithi022.stderr:sudo: ntpd: command not found 2024-09-13T22:57:51.728 INFO:teuthology.orchestra.run.smithi022.stdout:506 Cannot talk to daemon 2024-09-13T22:57:51.737 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: ntpd: command not found 2024-09-13T22:57:51.743 INFO:teuthology.orchestra.run.smithi022.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T22:57:51.748 INFO:teuthology.orchestra.run.smithi076.stdout:506 Cannot talk to daemon 2024-09-13T22:57:51.759 INFO:teuthology.orchestra.run.smithi022.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T22:57:51.763 INFO:teuthology.orchestra.run.smithi076.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T22:57:51.778 INFO:teuthology.orchestra.run.smithi076.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T22:57:51.816 INFO:teuthology.orchestra.run.smithi022.stderr:bash: line 1: ntpq: command not found 2024-09-13T22:57:51.820 INFO:teuthology.orchestra.run.smithi022.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T22:57:51.820 INFO:teuthology.orchestra.run.smithi022.stdout:=============================================================================== 2024-09-13T22:57:51.821 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:57:51.821 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:57:51.821 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:57:51.821 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:57:51.834 INFO:teuthology.orchestra.run.smithi076.stderr:bash: line 1: ntpq: command not found 2024-09-13T22:57:51.837 INFO:teuthology.orchestra.run.smithi076.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T22:57:51.837 INFO:teuthology.orchestra.run.smithi076.stdout:=============================================================================== 2024-09-13T22:57:51.837 INFO:teuthology.orchestra.run.smithi076.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:57:51.837 INFO:teuthology.orchestra.run.smithi076.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:57:51.837 INFO:teuthology.orchestra.run.smithi076.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:57:51.837 INFO:teuthology.orchestra.run.smithi076.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:57:51.838 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-13T22:57:51.934 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_STRAY_DAEMON', 'CEPHADM_FAILED_DAEMON', 'CEPHADM_AGENT_DOWN'], 'log-only-match': ['CEPHADM_'], 'sha1': 'ecab7be58963a5bd03910cd73eef66405d7fdd50'} 2024-09-13T22:57:51.934 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-13T22:57:51.935 INFO:tasks.cephadm:Cluster fsid is 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:57:51.935 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-13T22:57:51.935 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.22', 'mon.c': '[v2:172.21.15.22:3301,v1:172.21.15.22:6790]', 'mon.b': '172.21.15.76'} 2024-09-13T22:57:51.935 INFO:tasks.cephadm:First mon is mon.a on smithi022 2024-09-13T22:57:51.935 INFO:tasks.cephadm:First mgr is y 2024-09-13T22:57:51.935 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-13T22:57:51.936 DEBUG:teuthology.orchestra.run.smithi022:> sudo hostname $(hostname -s) 2024-09-13T22:57:51.963 DEBUG:teuthology.orchestra.run.smithi076:> sudo hostname $(hostname -s) 2024-09-13T22:57:51.989 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-13T22:57:51.989 DEBUG:teuthology.orchestra.run.smithi022:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T22:57:52.187 INFO:teuthology.orchestra.run.smithi022.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 13 22:57 /home/ubuntu/cephtest/cephadm 2024-09-13T22:57:52.187 DEBUG:teuthology.orchestra.run.smithi076:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T22:57:52.398 INFO:teuthology.orchestra.run.smithi076.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 13 22:57 /home/ubuntu/cephtest/cephadm 2024-09-13T22:57:52.398 DEBUG:teuthology.orchestra.run.smithi022:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T22:57:52.416 DEBUG:teuthology.orchestra.run.smithi076:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T22:57:52.443 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-13T22:57:52.444 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-13T22:57:52.459 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-13T22:57:54.135 INFO:teuthology.orchestra.run.smithi076.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T22:57:54.185 INFO:teuthology.orchestra.run.smithi022.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T22:58:05.698 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-13T22:58:05.698 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-13T22:58:05.698 INFO:teuthology.orchestra.run.smithi076.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-13T22:58:05.699 INFO:teuthology.orchestra.run.smithi076.stdout: "repo_digests": [ 2024-09-13T22:58:05.699 INFO:teuthology.orchestra.run.smithi076.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-13T22:58:05.699 INFO:teuthology.orchestra.run.smithi076.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-13T22:58:05.699 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-13T22:58:05.699 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-13T22:58:06.975 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T22:58:06.975 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-13T22:58:06.975 INFO:teuthology.orchestra.run.smithi022.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-13T22:58:06.975 INFO:teuthology.orchestra.run.smithi022.stdout: "repo_digests": [ 2024-09-13T22:58:06.975 INFO:teuthology.orchestra.run.smithi022.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-13T22:58:06.975 INFO:teuthology.orchestra.run.smithi022.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-13T22:58:06.975 INFO:teuthology.orchestra.run.smithi022.stdout: ] 2024-09-13T22:58:06.975 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T22:58:06.998 DEBUG:teuthology.orchestra.run.smithi022:> sudo mkdir -p /etc/ceph 2024-09-13T22:58:07.039 DEBUG:teuthology.orchestra.run.smithi076:> sudo mkdir -p /etc/ceph 2024-09-13T22:58:07.071 DEBUG:teuthology.orchestra.run.smithi022:> sudo chmod 777 /etc/ceph 2024-09-13T22:58:07.104 DEBUG:teuthology.orchestra.run.smithi076:> sudo chmod 777 /etc/ceph 2024-09-13T22:58:07.140 INFO:tasks.cephadm:Writing seed config... 2024-09-13T22:58:07.141 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-09-13T22:58:07.141 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-13T22:58:07.141 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-13T22:58:07.141 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-09-13T22:58:07.141 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-13T22:58:07.141 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-13T22:58:07.141 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-13T22:58:07.141 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-13T22:58:07.141 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-13T22:58:07.142 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-13T22:58:07.142 DEBUG:teuthology.orchestra.run.smithi022:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-13T22:58:07.159 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 9adcac8c-7223-11ef-bceb-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-13T22:58:07.160 DEBUG:teuthology.orchestra.run.smithi022:mon.a> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.a.service 2024-09-13T22:58:07.201 DEBUG:teuthology.orchestra.run.smithi022:mgr.y> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.y.service 2024-09-13T22:58:07.244 INFO:tasks.cephadm:Bootstrapping... 2024-09-13T22:58:07.244 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.22 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:58:07.453 INFO:teuthology.orchestra.run.smithi022.stderr:-------------------------------------------------------------------------------- 2024-09-13T22:58:07.453 INFO:teuthology.orchestra.run.smithi022.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '9adcac8c-7223-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.22', '--skip-admin-label'] 2024-09-13T22:58:07.476 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: 5.2.2 2024-09-13T22:58:07.478 INFO:teuthology.orchestra.run.smithi022.stderr:Verifying podman|docker is present... 2024-09-13T22:58:07.499 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: 5.2.2 2024-09-13T22:58:07.501 INFO:teuthology.orchestra.run.smithi022.stderr:Verifying lvm2 is present... 2024-09-13T22:58:07.501 INFO:teuthology.orchestra.run.smithi022.stderr:Verifying time synchronization is in place... 2024-09-13T22:58:07.508 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T22:58:07.515 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: inactive 2024-09-13T22:58:07.522 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: enabled 2024-09-13T22:58:07.529 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: active 2024-09-13T22:58:07.529 INFO:teuthology.orchestra.run.smithi022.stderr:Unit chronyd.service is enabled and running 2024-09-13T22:58:07.529 INFO:teuthology.orchestra.run.smithi022.stderr:Repeating the final host check... 2024-09-13T22:58:07.549 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: 5.2.2 2024-09-13T22:58:07.552 INFO:teuthology.orchestra.run.smithi022.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-13T22:58:07.552 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl is present 2024-09-13T22:58:07.552 INFO:teuthology.orchestra.run.smithi022.stderr:lvcreate is present 2024-09-13T22:58:07.558 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T22:58:07.564 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: inactive 2024-09-13T22:58:07.571 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: enabled 2024-09-13T22:58:07.578 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: active 2024-09-13T22:58:07.580 INFO:teuthology.orchestra.run.smithi022.stderr:Unit chronyd.service is enabled and running 2024-09-13T22:58:07.580 INFO:teuthology.orchestra.run.smithi022.stderr:Host looks OK 2024-09-13T22:58:07.580 INFO:teuthology.orchestra.run.smithi022.stderr:Cluster fsid: 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:58:07.580 INFO:teuthology.orchestra.run.smithi022.stderr:Acquiring lock 140699552633472 on /run/cephadm/9adcac8c-7223-11ef-bceb-c7b262605968.lock 2024-09-13T22:58:07.580 INFO:teuthology.orchestra.run.smithi022.stderr:Lock 140699552633472 acquired on /run/cephadm/9adcac8c-7223-11ef-bceb-c7b262605968.lock 2024-09-13T22:58:07.580 INFO:teuthology.orchestra.run.smithi022.stderr:Verifying IP 172.21.15.22 port 3300 ... 2024-09-13T22:58:07.580 INFO:teuthology.orchestra.run.smithi022.stderr:Verifying IP 172.21.15.22 port 6789 ... 2024-09-13T22:58:07.580 INFO:teuthology.orchestra.run.smithi022.stderr:Base mon IP is 172.21.15.22, final addrv is [v2:172.21.15.22:3300,v1:172.21.15.22:6789] 2024-09-13T22:58:07.582 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.22 metric 100 2024-09-13T22:58:07.582 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.22 metric 100 2024-09-13T22:58:07.585 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-13T22:58:07.585 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-13T22:58:07.585 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1798sec hoplimit 64 pref medium 2024-09-13T22:58:07.588 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-13T22:58:07.588 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-13T22:58:07.588 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-13T22:58:07.588 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-13T22:58:07.588 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe59:5de4/64 scope link 2024-09-13T22:58:07.588 INFO:teuthology.orchestra.run.smithi022.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-13T22:58:07.589 INFO:teuthology.orchestra.run.smithi022.stderr:Mon IP `172.21.15.22` is in CIDR network `172.21.0.0/20` 2024-09-13T22:58:07.589 INFO:teuthology.orchestra.run.smithi022.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-13T22:58:07.589 INFO:teuthology.orchestra.run.smithi022.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T22:58:07.718 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-13T22:58:08.083 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Getting image source signatures 2024-09-13T22:58:08.084 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-13T22:58:08.084 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-13T22:58:08.084 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-13T22:58:08.084 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-13T22:58:08.084 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-13T22:58:08.084 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-13T22:58:08.401 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: Writing manifest to image destination 2024-09-13T22:58:08.521 INFO:teuthology.orchestra.run.smithi022.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-13T22:58:08.922 INFO:teuthology.orchestra.run.smithi022.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-13T22:58:09.461 INFO:teuthology.orchestra.run.smithi022.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-13T22:58:09.461 INFO:teuthology.orchestra.run.smithi022.stderr:Extracting ceph user uid/gid from container image... 2024-09-13T22:58:12.014 INFO:teuthology.orchestra.run.smithi022.stderr:stat: 167 167 2024-09-13T22:58:17.675 INFO:teuthology.orchestra.run.smithi022.stderr:Creating initial keys... 2024-09-13T22:58:20.834 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph-authtool: AQCMw+RmM0gVFBAAQhBgzZDiuom5rtcxkxkmJQ== 2024-09-13T22:58:21.689 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph-authtool: AQCNw+RmB8gUJRAA7QQ4K3e26ZnHT/dFWDyj8w== 2024-09-13T22:58:22.636 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph-authtool: AQCOw+RmOkb0IBAAHlr64GoOcFdv4cwgDn7zbg== 2024-09-13T22:58:23.769 INFO:teuthology.orchestra.run.smithi022.stderr:Creating initial monmap... 2024-09-13T22:58:24.184 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T22:58:24.184 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-13T22:58:24.184 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:58:24.184 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T22:58:24.691 INFO:teuthology.orchestra.run.smithi022.stderr:monmaptool for a [v2:172.21.15.22:3300,v1:172.21.15.22:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T22:58:24.691 INFO:teuthology.orchestra.run.smithi022.stderr:setting min_mon_release = octopus 2024-09-13T22:58:24.691 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/monmaptool: set fsid to 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:58:24.691 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T22:58:24.691 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-09-13T22:58:24.691 INFO:teuthology.orchestra.run.smithi022.stderr:Creating mon... 2024-09-13T22:58:25.629 INFO:teuthology.orchestra.run.smithi022.stderr:create mon.a on 2024-09-13T22:58:25.862 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-13T22:58:26.180 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-9adcac8c-7223-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-9adcac8c-7223-11ef-bceb-c7b262605968.target. 2024-09-13T22:58:26.180 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-9adcac8c-7223-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-9adcac8c-7223-11ef-bceb-c7b262605968.target. 2024-09-13T22:58:26.485 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Failed to reset failed state of unit ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-13T22:58:26.492 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9adcac8c-7223-11ef-bceb-c7b262605968.target.wants/ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.a.service → /etc/systemd/system/ceph-9adcac8c-7223-11ef-bceb-c7b262605968@.service. 2024-09-13T22:58:27.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:27 smithi022 ceph-mon[23407]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T22:58:27.175 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: disabled 2024-09-13T22:58:27.183 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: inactive 2024-09-13T22:58:27.184 INFO:teuthology.orchestra.run.smithi022.stderr:firewalld.service is not enabled 2024-09-13T22:58:27.184 INFO:teuthology.orchestra.run.smithi022.stderr:Not possible to enable service . firewalld.service is not available 2024-09-13T22:58:27.184 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for mon to start... 2024-09-13T22:58:27.185 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for mon... 2024-09-13T22:58:27.403 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:27 smithi022 bash[23388]: c765345bf7b8294fb5465c3252516e6beb4e0d4659dacd839bc2bf9199aef588 2024-09-13T22:58:27.403 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:27 smithi022 systemd[1]: Started Ceph mon.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T22:58:27.704 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: cluster: 2024-09-13T22:58:27.704 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: id: 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:58:27.704 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-13T22:58:27.704 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-09-13T22:58:27.704 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: services: 2024-09-13T22:58:27.705 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.595737s) 2024-09-13T22:58:27.705 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-13T22:58:27.705 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-13T22:58:27.705 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-09-13T22:58:27.705 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: data: 2024-09-13T22:58:27.705 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-13T22:58:27.705 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-13T22:58:27.705 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-13T22:58:27.705 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: pgs: 2024-09-13T22:58:27.705 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-09-13T22:58:28.291 INFO:teuthology.orchestra.run.smithi022.stderr:mon is available 2024-09-13T22:58:28.291 INFO:teuthology.orchestra.run.smithi022.stderr:Assimilating anything we can from ceph.conf... 2024-09-13T22:58:28.380 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:28 smithi022 ceph-mon[23407]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T22:58:28.380 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:28 smithi022 ceph-mon[23407]: monmap e1: 1 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0]} 2024-09-13T22:58:28.380 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:28 smithi022 ceph-mon[23407]: fsmap 2024-09-13T22:58:28.380 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:28 smithi022 ceph-mon[23407]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T22:58:28.380 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:28 smithi022 ceph-mon[23407]: mgrmap e1: no daemons active 2024-09-13T22:58:28.380 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:28 smithi022 ceph-mon[23407]: from='client.? 172.21.15.22:0/908736449' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T22:58:28.821 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-09-13T22:58:28.821 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: [global] 2024-09-13T22:58:28.821 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: fsid = 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:58:28.821 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.22:3300,v1:172.21.15.22:6789] 2024-09-13T22:58:28.821 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-13T22:58:28.821 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-13T22:58:28.822 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-13T22:58:28.822 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-13T22:58:28.822 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-09-13T22:58:28.822 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: [mgr] 2024-09-13T22:58:28.822 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-09-13T22:58:28.822 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-13T22:58:28.822 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-09-13T22:58:28.822 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: [osd] 2024-09-13T22:58:28.822 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-13T22:58:28.822 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-13T22:58:29.370 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:29 smithi022 ceph-mon[23407]: from='client.? 172.21.15.22:0/2868679168' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T22:58:29.370 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:29 smithi022 ceph-mon[23407]: from='client.? 172.21.15.22:0/2868679168' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T22:58:29.372 INFO:teuthology.orchestra.run.smithi022.stderr:Generating new minimal ceph.conf... 2024-09-13T22:58:30.384 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:30 smithi022 ceph-mon[23407]: from='client.? 172.21.15.22:0/264916777' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:58:30.387 INFO:teuthology.orchestra.run.smithi022.stderr:Restarting the monitor... 2024-09-13T22:58:30.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:30 smithi022 systemd[1]: Stopping Ceph mon.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T22:58:30.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:30 smithi022 bash[23637]: Error: no container with name or ID "ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon.a" found: no such container 2024-09-13T22:58:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:30 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a[23403]: 2024-09-13T22:58:30.545+0000 7f1d6af97700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:58:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:30 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a[23403]: 2024-09-13T22:58:30.545+0000 7f1d6af97700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-13T22:58:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:30 smithi022 podman[23648]: 2024-09-13 22:58:30.627530471 +0000 UTC m=+0.183900369 container died c765345bf7b8294fb5465c3252516e6beb4e0d4659dacd839bc2bf9199aef588 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, release=754, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.expose-services=, version=8, GIT_CLEAN=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, vcs-type=git, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T22:58:31.139 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:30 smithi022 podman[23648]: 2024-09-13 22:58:30.970068824 +0000 UTC m=+0.526438717 container cleanup c765345bf7b8294fb5465c3252516e6beb4e0d4659dacd839bc2bf9199aef588 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, version=8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.openshift.expose-services=, name=centos-stream, architecture=x86_64, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, release=754, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public) 2024-09-13T22:58:31.139 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:30 smithi022 bash[23648]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a 2024-09-13T22:58:31.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 podman[23662]: 2024-09-13 22:58:31.137627642 +0000 UTC m=+0.501982927 container remove c765345bf7b8294fb5465c3252516e6beb4e0d4659dacd839bc2bf9199aef588 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_CLEAN=True, vcs-type=git, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vendor=Red Hat, Inc., RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, name=centos-stream, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T22:58:31.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 bash[23673]: Error: no container with name or ID "ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon.a" found: no such container 2024-09-13T22:58:31.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T22:58:31.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 systemd[1]: Stopped Ceph mon.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T22:58:31.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 systemd[1]: Starting Ceph mon.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T22:58:31.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 podman[23739]: 2024-09-13 22:58:31.413532273 +0000 UTC m=+0.110217672 container create 7d59bd9c71e84aec45b371b4e13f145608d3160c519df781040f2901358d50c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=-17.2.0, release=754, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, distribution-scope=public, io.openshift.tags=base centos centos-stream, vcs-type=git, vendor=Red Hat, Inc., name=centos-stream) 2024-09-13T22:58:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 podman[23739]: 2024-09-13 22:58:31.32050935 +0000 UTC m=+0.017194712 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-13T22:58:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 podman[23739]: 2024-09-13 22:58:31.657525916 +0000 UTC m=+0.354211264 container init 7d59bd9c71e84aec45b371b4e13f145608d3160c519df781040f2901358d50c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, release=754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.openshift.expose-services=, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, RELEASE=HEAD) 2024-09-13T22:58:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 podman[23739]: 2024-09-13 22:58:31.661177891 +0000 UTC m=+0.357863239 container start 7d59bd9c71e84aec45b371b4e13f145608d3160c519df781040f2901358d50c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.openshift.expose-services=, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.buildah.version=1.19.8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, version=8) 2024-09-13T22:58:31.760 INFO:teuthology.orchestra.run.smithi022.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-13T22:58:31.978 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T22:58:31.978 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T22:58:31.978 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: pidfile_write: ignore empty --pid-file 2024-09-13T22:58:31.978 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: load: jerasure load: lrc 2024-09-13T22:58:31.979 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: RocksDB version: 6.15.5 2024-09-13T22:58:31.979 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T22:58:31.979 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Compile date Apr 18 2022 2024-09-13T22:58:31.979 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: DB SUMMARY 2024-09-13T22:58:31.979 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: DB Session ID: BIPEDDG9MFQ9OFAZW502 2024-09-13T22:58:31.979 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: CURRENT file: CURRENT 2024-09-13T22:58:31.979 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T22:58:31.979 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-13T22:58:31.979 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-13T22:58:31.979 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82627 ; 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.error_if_exists: 0 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.create_if_missing: 0 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.paranoid_checks: 1 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.env: 0x5591516c6860 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.fs: Posix File System 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.info_log: 0x5591536a3dc0 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.statistics: (nil) 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.use_fsync: 0 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_log_file_size: 0 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T22:58:31.980 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.allow_fallocate: 1 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.use_direct_reads: 0 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.db_log_dir: 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T22:58:31.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.write_buffer_manager: 0x5591537942a0 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T22:58:31.982 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T22:58:31.983 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T22:58:31.983 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T22:58:31.983 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.unordered_write: 0 2024-09-13T22:58:31.983 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T22:58:31.983 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T22:58:31.983 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T22:58:31.983 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T22:58:31.983 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.row_cache: None 2024-09-13T22:58:31.983 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.wal_filter: None 2024-09-13T22:58:31.983 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T22:58:31.983 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T22:58:31.983 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.preserve_deletes: 0 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.two_write_queues: 0 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.atomic_flush: 0 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.log_readahead_size: 0 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T22:58:31.984 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T22:58:31.985 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_background_jobs: 2 2024-09-13T22:58:31.986 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_background_compactions: -1 2024-09-13T22:58:31.986 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_subcompactions: 1 2024-09-13T22:58:31.986 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T22:58:31.986 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T22:58:31.986 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T22:58:31.986 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T22:58:31.986 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T22:58:31.986 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T22:58:31.986 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T22:58:31.986 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T22:58:31.986 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_open_files: -1 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_background_flushes: -1 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Compression algorithms supported: 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: kZSTD supported: 0 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: kXpressCompression supported: 0 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: kLZ4Compression supported: 1 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: kBZip2Compression supported: 0 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: kZlibCompression supported: 1 2024-09-13T22:58:31.987 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: kSnappyCompression supported: 1 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.merge_operator: 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_filter: None 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55915366fd00) 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout: cache_index_and_filter_blocks: 1 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T22:58:31.988 INFO:journalctl@ceph.mon.a.smithi022.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: pin_top_level_index_and_filter: 1 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: index_type: 0 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: data_block_index_type: 0 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: index_shortening: 1 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: hash_index_allow_collision: 1 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: checksum: 1 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: no_block_cache: 0 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache: 0x5591536da170 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache_name: BinnedLRUCache 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache_options: 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: capacity : 536870912 2024-09-13T22:58:31.989 INFO:journalctl@ceph.mon.a.smithi022.stdout: num_shard_bits : 4 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: strict_capacity_limit : 0 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: high_pri_pool_ratio: 0.000 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache_compressed: (nil) 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: persistent_cache: (nil) 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_size: 4096 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_size_deviation: 10 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_restart_interval: 16 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: index_block_restart_interval: 1 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: metadata_block_size: 4096 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: partition_filters: 0 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: use_delta_encoding: 1 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T22:58:31.990 INFO:journalctl@ceph.mon.a.smithi022.stdout: whole_key_filtering: 1 2024-09-13T22:58:31.991 INFO:journalctl@ceph.mon.a.smithi022.stdout: verify_compression: 0 2024-09-13T22:58:31.991 INFO:journalctl@ceph.mon.a.smithi022.stdout: read_amp_bytes_per_bit: 0 2024-09-13T22:58:31.991 INFO:journalctl@ceph.mon.a.smithi022.stdout: format_version: 4 2024-09-13T22:58:31.991 INFO:journalctl@ceph.mon.a.smithi022.stdout: enable_index_compression: 1 2024-09-13T22:58:31.991 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_align: 0 2024-09-13T22:58:31.991 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T22:58:31.991 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T22:58:31.991 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compression: NoCompression 2024-09-13T22:58:31.991 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T22:58:31.991 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T22:58:31.991 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T22:58:31.991 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.num_levels: 7 2024-09-13T22:58:31.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T22:58:31.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T22:58:31.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T22:58:31.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T22:58:31.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T22:58:31.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T22:58:31.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T22:58:31.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:58:31.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T22:58:31.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T22:58:31.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T22:58:31.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T22:58:31.993 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T22:58:31.993 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T22:58:31.993 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:58:31.993 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T22:58:31.993 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T22:58:31.993 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T22:58:31.993 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T22:58:31.993 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T22:58:31.993 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T22:58:31.993 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T22:58:31.993 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T22:58:31.993 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T22:58:31.994 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T22:58:31.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.table_properties_collectors: 2024-09-13T22:58:31.996 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.inplace_update_support: 0 2024-09-13T22:58:31.996 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T22:58:31.996 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T22:58:31.996 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T22:58:31.996 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T22:58:31.996 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.bloom_locality: 0 2024-09-13T22:58:31.996 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.max_successive_merges: 0 2024-09-13T22:58:31.996 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T22:58:31.996 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T22:58:31.996 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T22:58:31.996 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T22:58:31.996 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.ttl: 2592000 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.enable_blob_files: false 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.min_blob_size: 0 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726268311688369, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-13T22:58:31.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T22:58:31.998 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726268311689128, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79177, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77440, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71749, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726268311, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "ef30e49d-f9e9-43be-9d2f-9ecaecbea880", "db_session_id": "BIPEDDG9MFQ9OFAZW502"}} 2024-09-13T22:58:31.998 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-13T22:58:31.998 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726268311689469, "job": 1, "event": "recovery_finished"} 2024-09-13T22:58:31.998 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T22:58:31.998 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5591536c0a80 2024-09-13T22:58:31.998 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: DB pointer 0x5591536d0000 2024-09-13T22:58:31.998 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T22:58:31.998 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T22:58:31.998 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** DB Stats ** 2024-09-13T22:58:31.998 INFO:journalctl@ceph.mon.a.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:58:31.998 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T22:58:31.999 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:58:31.999 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:58:31.999 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T22:58:31.999 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T22:58:31.999 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:58:31.999 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T22:58:31.999 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-13T22:58:31.999 INFO:journalctl@ceph.mon.a.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:58:31.999 INFO:journalctl@ceph.mon.a.smithi022.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:58:31.999 INFO:journalctl@ceph.mon.a.smithi022.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:58:31.999 INFO:journalctl@ceph.mon.a.smithi022.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:58:31.999 INFO:journalctl@ceph.mon.a.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:58:32.000 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T22:58:32.000 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-13T22:58:32.000 INFO:journalctl@ceph.mon.a.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:58:32.000 INFO:journalctl@ceph.mon.a.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:58:32.000 INFO:journalctl@ceph.mon.a.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:58:32.000 INFO:journalctl@ceph.mon.a.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:58:32.000 INFO:journalctl@ceph.mon.a.smithi022.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:58:32.000 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:58:32.000 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:58:32.000 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:58:32.000 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:58:32.000 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative compaction: 0.00 GB write, 24.25 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval compaction: 0.00 GB write, 24.25 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-13T22:58:32.001 INFO:journalctl@ceph.mon.a.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:58:32.002 INFO:journalctl@ceph.mon.a.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:58:32.002 INFO:journalctl@ceph.mon.a.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:58:32.002 INFO:journalctl@ceph.mon.a.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:58:32.002 INFO:journalctl@ceph.mon.a.smithi022.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:58:32.002 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:58:32.002 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:58:32.002 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:58:32.002 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:58:32.002 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative compaction: 0.00 GB write, 24.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:58:32.002 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:58:32.002 INFO:journalctl@ceph.mon.a.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:58:32.002 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: starting mon.a rank 0 at public addrs [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] at bind addrs [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: mon.a@-1(???) e1 preinit fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: mon.a@-1(???).mds e1 new map 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: mon.a@-1(???).mds e1 print_map 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout: e1 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout: legacy client fscid: -1 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout: No filesystems configured 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T22:58:32.003 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:58:32.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:58:32.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:58:32.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-13T22:58:32.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T22:58:32.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: monmap e1: 1 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0]} 2024-09-13T22:58:32.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: fsmap 2024-09-13T22:58:32.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T22:58:32.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 ceph-mon[23758]: mgrmap e1: no daemons active 2024-09-13T22:58:32.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 bash[23739]: 7d59bd9c71e84aec45b371b4e13f145608d3160c519df781040f2901358d50c1 2024-09-13T22:58:32.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:31 smithi022 systemd[1]: Started Ceph mon.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T22:58:32.801 INFO:teuthology.orchestra.run.smithi022.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-13T22:58:32.801 INFO:teuthology.orchestra.run.smithi022.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:58:32.801 INFO:teuthology.orchestra.run.smithi022.stderr:Creating mgr... 2024-09-13T22:58:32.801 INFO:teuthology.orchestra.run.smithi022.stderr:Verifying port 9283 ... 2024-09-13T22:58:32.982 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Failed to reset failed state of unit ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-13T22:58:32.990 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9adcac8c-7223-11ef-bceb-c7b262605968.target.wants/ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-9adcac8c-7223-11ef-bceb-c7b262605968@.service. 2024-09-13T22:58:33.258 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:33 smithi022 systemd[1]: Starting Ceph mgr.y for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T22:58:33.514 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:33 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3073145017' entity='client.admin' 2024-09-13T22:58:33.514 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:33 smithi022 podman[23971]: 2024-09-13 22:58:33.278687538 +0000 UTC m=+0.016646922 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-13T22:58:33.514 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:33 smithi022 podman[23971]: 2024-09-13 22:58:33.381577058 +0000 UTC m=+0.119536445 container create 07f20d2b59b79619fc705742b216871bf40bbb8e763d6b1e4835da7f7ee5501f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.buildah.version=1.19.8, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public) 2024-09-13T22:58:33.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:33 smithi022 podman[23971]: 2024-09-13 22:58:33.600726413 +0000 UTC m=+0.338685798 container init 07f20d2b59b79619fc705742b216871bf40bbb8e763d6b1e4835da7f7ee5501f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, version=8, release=754, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, ceph=True, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, io.openshift.expose-services=) 2024-09-13T22:58:33.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:33 smithi022 podman[23971]: 2024-09-13 22:58:33.603908818 +0000 UTC m=+0.341868205 container start 07f20d2b59b79619fc705742b216871bf40bbb8e763d6b1e4835da7f7ee5501f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, release=754, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-13T22:58:33.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:33.750+0000 7f03a8a05000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:58:33.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:33.830+0000 7f03a8a05000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:58:33.994 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: disabled 2024-09-13T22:58:34.006 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: inactive 2024-09-13T22:58:34.007 INFO:teuthology.orchestra.run.smithi022.stderr:firewalld.service is not enabled 2024-09-13T22:58:34.007 INFO:teuthology.orchestra.run.smithi022.stderr:Not possible to enable service . firewalld.service is not available 2024-09-13T22:58:34.016 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: disabled 2024-09-13T22:58:34.023 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: inactive 2024-09-13T22:58:34.024 INFO:teuthology.orchestra.run.smithi022.stderr:firewalld.service is not enabled 2024-09-13T22:58:34.024 INFO:teuthology.orchestra.run.smithi022.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-13T22:58:34.024 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for mgr to start... 2024-09-13T22:58:34.024 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for mgr... 2024-09-13T22:58:34.258 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:33 smithi022 bash[23971]: 07f20d2b59b79619fc705742b216871bf40bbb8e763d6b1e4835da7f7ee5501f 2024-09-13T22:58:34.259 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:33 smithi022 systemd[1]: Started Ceph mgr.y for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T22:58:34.259 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:34.026+0000 7f03a8a05000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:58:34.259 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:34.121+0000 7f03a8a05000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:58:34.259 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:34.258+0000 7f03a8a05000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:58:34.528 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:34.474+0000 7f03a8a05000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:58:34.922 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-09-13T22:58:34.922 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-09-13T22:58:34.922 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "fsid": "9adcac8c-7223-11ef-bceb-c7b262605968", 2024-09-13T22:58:34.922 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "health": { 2024-09-13T22:58:34.923 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T22:58:34.923 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T22:58:34.923 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T22:58:34.923 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:34.923 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T22:58:34.923 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T22:58:34.923 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 0 2024-09-13T22:58:34.923 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "a" 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum_age": 3, 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T22:58:34.924 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T22:58:34.925 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T22:58:34.928 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T22:58:34.928 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T22:58:34.928 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:34.928 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T22:58:34.929 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T22:58:34.929 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T22:58:34.929 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T22:58:34.929 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T22:58:34.929 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T22:58:34.929 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T22:58:34.929 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T22:58:34.929 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T22:58:34.929 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:34.929 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T22:58:34.929 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:58:34.929 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "available": false, 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "iostat", 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "nfs", 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "restful" 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:34.930 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T22:58:34.931 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:58:34.931 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "modified": "2024-09-13T22:58:27.106694+0000", 2024-09-13T22:58:34.931 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:58:34.931 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:34.931 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T22:58:34.931 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-09-13T22:58:35.096 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:34 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1189506077' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T22:58:35.096 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:34.826+0000 7f03a8a05000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:58:35.097 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:34.918+0000 7f03a8a05000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:58:35.097 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:35.096+0000 7f03a8a05000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:58:35.527 INFO:teuthology.orchestra.run.smithi022.stderr:mgr not available, waiting (1/15)... 2024-09-13T22:58:36.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:36 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:36.198+0000 7f03a8a05000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:58:36.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:36 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:36.292+0000 7f03a8a05000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:58:37.557 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:37 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:37.239+0000 7f03a8a05000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:58:37.557 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:37 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:37.556+0000 7f03a8a05000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:58:37.890 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:37 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:37.695+0000 7f03a8a05000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:58:37.890 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:37 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:37.789+0000 7f03a8a05000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:58:38.076 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-09-13T22:58:38.076 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-09-13T22:58:38.076 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "fsid": "9adcac8c-7223-11ef-bceb-c7b262605968", 2024-09-13T22:58:38.076 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "health": { 2024-09-13T22:58:38.076 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T22:58:38.076 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T22:58:38.076 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T22:58:38.076 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:38.076 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T22:58:38.076 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T22:58:38.077 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 0 2024-09-13T22:58:38.077 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-09-13T22:58:38.077 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T22:58:38.077 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "a" 2024-09-13T22:58:38.077 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-09-13T22:58:38.077 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-09-13T22:58:38.077 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T22:58:38.077 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:58:38.077 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T22:58:38.077 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T22:58:38.077 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:38.077 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T22:58:38.078 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:58:38.078 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T22:58:38.078 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T22:58:38.078 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T22:58:38.078 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T22:58:38.078 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T22:58:38.078 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T22:58:38.078 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:38.078 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T22:58:38.078 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T22:58:38.078 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T22:58:38.078 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T22:58:38.079 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "available": false, 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "iostat", 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "nfs", 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "restful" 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "modified": "2024-09-13T22:58:27.106694+0000", 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:58:38.080 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:38.081 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T22:58:38.081 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-09-13T22:58:38.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:38 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3258845055' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T22:58:38.144 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:37 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:37.890+0000 7f03a8a05000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:58:38.144 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:37 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:37.983+0000 7f03a8a05000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:58:38.145 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:38 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:38.143+0000 7f03a8a05000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:58:38.426 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:38 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:38.242+0000 7f03a8a05000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:58:38.426 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:38 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:38.342+0000 7f03a8a05000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:58:38.728 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:38 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:38.637+0000 7f03a8a05000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:58:38.728 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:38 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:38.728+0000 7f03a8a05000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:58:38.746 INFO:teuthology.orchestra.run.smithi022.stderr:mgr not available, waiting (2/15)... 2024-09-13T22:58:39.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:38 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:38.820+0000 7f03a8a05000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:58:40.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:39.919+0000 7f03a8a05000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:58:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: Activating manager daemon y 2024-09-13T22:58:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: mgrmap e2: y(active, starting, since 0.00273508s) 2024-09-13T22:58:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:58:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:58:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:58:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:58:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:58:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: Manager daemon y is now available 2024-09-13T22:58:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:58:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:58:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' 2024-09-13T22:58:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' 2024-09-13T22:58:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:39 smithi022 ceph-mon[23758]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' 2024-09-13T22:58:41.440 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-09-13T22:58:41.440 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-09-13T22:58:41.440 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "fsid": "9adcac8c-7223-11ef-bceb-c7b262605968", 2024-09-13T22:58:41.440 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "health": { 2024-09-13T22:58:41.440 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T22:58:41.440 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T22:58:41.440 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T22:58:41.440 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:41.440 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 0 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "a" 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T22:58:41.441 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:58:41.442 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T22:58:41.442 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T22:58:41.442 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T22:58:41.442 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T22:58:41.445 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T22:58:41.445 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T22:58:41.445 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:41.445 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T22:58:41.446 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "available": true, 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "iostat", 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "nfs", 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "restful" 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ], 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:58:41.447 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "modified": "2024-09-13T22:58:27.106694+0000", 2024-09-13T22:58:41.448 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:58:41.448 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: }, 2024-09-13T22:58:41.448 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T22:58:41.448 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-09-13T22:58:42.166 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:41 smithi022 ceph-mon[23758]: mgrmap e3: y(active, since 1.00704s) 2024-09-13T22:58:42.167 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:41 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1532965570' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T22:58:42.169 INFO:teuthology.orchestra.run.smithi022.stderr:mgr is available 2024-09-13T22:58:42.781 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-09-13T22:58:42.781 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: [global] 2024-09-13T22:58:42.781 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: fsid = 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:58:42.781 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-13T22:58:42.781 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-13T22:58:42.781 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-13T22:58:42.781 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-13T22:58:42.781 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-09-13T22:58:42.781 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: [mgr] 2024-09-13T22:58:42.781 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-13T22:58:42.782 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 2024-09-13T22:58:42.782 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: [osd] 2024-09-13T22:58:42.782 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-13T22:58:42.782 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-13T22:58:43.367 INFO:teuthology.orchestra.run.smithi022.stderr:Enabling cephadm module... 2024-09-13T22:58:43.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:43 smithi022 ceph-mon[23758]: mgrmap e4: y(active, since 2s) 2024-09-13T22:58:43.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:43 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3911960426' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T22:58:43.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:43 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3911960426' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T22:58:44.955 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:44 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3894662652' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T22:58:44.955 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:44 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ignoring --setuser ceph since I am not root 2024-09-13T22:58:44.955 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:44 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ignoring --setgroup ceph since I am not root 2024-09-13T22:58:44.955 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:44 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:44.873+0000 7fd79f6d5000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:58:44.955 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:44 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:44.954+0000 7fd79f6d5000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:58:45.238 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:45 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:45.147+0000 7fd79f6d5000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:58:45.580 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:45 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:45.238+0000 7fd79f6d5000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:58:45.580 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:45 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:45.378+0000 7fd79f6d5000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:58:45.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:45 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:45.598+0000 7fd79f6d5000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:58:45.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:45 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3894662652' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T22:58:45.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:45 smithi022 ceph-mon[23758]: mgrmap e5: y(active, since 4s) 2024-09-13T22:58:46.037 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-09-13T22:58:46.037 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-13T22:58:46.037 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "available": true, 2024-09-13T22:58:46.037 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "active_name": "y", 2024-09-13T22:58:46.037 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-13T22:58:46.038 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-09-13T22:58:46.211 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:45 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:45.945+0000 7fd79f6d5000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:58:46.211 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:46 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:46.036+0000 7fd79f6d5000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:58:46.211 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:46 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:46.211+0000 7fd79f6d5000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:58:46.596 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for the mgr to restart... 2024-09-13T22:58:46.596 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for mgr epoch 5... 2024-09-13T22:58:46.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:46 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1921923318' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T22:58:47.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:47 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:47.348+0000 7fd79f6d5000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:58:47.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:47 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:47.441+0000 7fd79f6d5000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:58:48.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:48 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:48.396+0000 7fd79f6d5000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:58:49.045 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:48 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:48.715+0000 7fd79f6d5000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:58:49.045 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:48 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:48.852+0000 7fd79f6d5000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:58:49.045 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:48 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:48.944+0000 7fd79f6d5000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:58:49.046 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:49 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:49.045+0000 7fd79f6d5000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:58:49.384 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:49 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:49.135+0000 7fd79f6d5000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:58:49.384 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:49 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:49.288+0000 7fd79f6d5000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:58:49.385 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:49 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:49.384+0000 7fd79f6d5000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:58:49.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:49 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:49.483+0000 7fd79f6d5000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:58:50.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:49 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:49.777+0000 7fd79f6d5000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:58:50.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:49 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:49.868+0000 7fd79f6d5000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:58:50.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:49 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:49.959+0000 7fd79f6d5000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:58:51.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:58:51.067+0000 7fd79f6d5000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:58:52.131 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-09-13T22:58:52.131 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-13T22:58:52.131 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "initialized": true 2024-09-13T22:58:52.131 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-09-13T22:58:52.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: Active manager daemon y restarted 2024-09-13T22:58:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: Activating manager daemon y 2024-09-13T22:58:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: osdmap e2: 0 total, 0 up, 0 in 2024-09-13T22:58:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: mgrmap e6: y(active, starting, since 0.0525028s) 2024-09-13T22:58:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:58:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:58:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:58:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:58:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:58:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: Manager daemon y is now available 2024-09-13T22:58:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:58:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:58:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:58:52.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:58:52.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:58:52.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:58:52.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:52 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:58:52.702 INFO:teuthology.orchestra.run.smithi022.stderr:mgr epoch 5 is available 2024-09-13T22:58:52.702 INFO:teuthology.orchestra.run.smithi022.stderr:Setting orchestrator backend to cephadm... 2024-09-13T22:58:53.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:53 smithi022 ceph-mon[23758]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T22:58:53.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:53 smithi022 ceph-mon[23758]: mgrmap e7: y(active, since 1.05503s) 2024-09-13T22:58:53.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:53 smithi022 ceph-mon[23758]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T22:58:53.613 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:22:58:53] ENGINE Bus STARTING 2024-09-13T22:58:53.613 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:22:58:53] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T22:58:53.613 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:22:58:53] ENGINE Bus STARTED 2024-09-13T22:58:54.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:54 smithi022 ceph-mon[23758]: mgrmap e8: y(active, since 2s) 2024-09-13T22:58:54.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:54 smithi022 ceph-mon[23758]: [13/Sep/2024:22:58:53] ENGINE Bus STARTING 2024-09-13T22:58:54.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:54 smithi022 ceph-mon[23758]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:58:54.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:54 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:58:54.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:54 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:58:54.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:54 smithi022 ceph-mon[23758]: [13/Sep/2024:22:58:53] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T22:58:54.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:54 smithi022 ceph-mon[23758]: [13/Sep/2024:22:58:53] ENGINE Bus STARTED 2024-09-13T22:58:54.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:54 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:58:54.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:54 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:58:54.590 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: value unchanged 2024-09-13T22:58:55.171 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:55 smithi022 ceph-mon[23758]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:58:55.174 INFO:teuthology.orchestra.run.smithi022.stderr:Generating ssh key... 2024-09-13T22:58:56.426 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: Generating public/private rsa key pair. 2024-09-13T22:58:56.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: Your identification has been saved in /tmp/tmpgye39_r5/key. 2024-09-13T22:58:56.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: Your public key has been saved in /tmp/tmpgye39_r5/key.pub. 2024-09-13T22:58:56.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: The key fingerprint is: 2024-09-13T22:58:56.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: SHA256:ZoqTO4xsLVQaF7YbENJByDGWM8Y1u1BoUE0GQUWui1Q ceph-9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:58:56.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: The key's randomart image is: 2024-09-13T22:58:56.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: +---[RSA 3072]----+ 2024-09-13T22:58:56.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: |*X&&= | 2024-09-13T22:58:56.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: |o@=o* | 2024-09-13T22:58:56.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: |o.oE.o | 2024-09-13T22:58:56.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: | +.* | 2024-09-13T22:58:56.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: | ..* o S | 2024-09-13T22:58:56.428 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: |..o..o + | 2024-09-13T22:58:56.428 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: |.o.++ . | 2024-09-13T22:58:56.428 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: | = +o | 2024-09-13T22:58:56.428 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: | . ... | 2024-09-13T22:58:56.428 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:58:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: +----[SHA256]-----+ 2024-09-13T22:58:57.114 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSH86XeZiumi+4hcnigksZzO/3+0CcN35SIxtAGMjFu5o/S1doQPBcXn64HjGfBj6tGKowMu40nTmy3LMN250KUxN2D3vNpLWiu7eY2ANP1EKts+qKWaHe4iEvuT2CJo+5N0etFn7tZf63JctFXHndRdcBjk3ZJ5dIRrEcwk+1MgNJnIm8DTiOUKtTrM0m+GkzRYt6FwzW5h837gWIu7vgxiEQawFxhn1PaMQb69nURI9zD9FXDnRLBPfOs5PDtpsQmMd0/t2PO9fqMrvFXUyXdMqIvue35W0Wri2ogF7tsCR1Z1cs57Maf8hRKgd3Eh66hogTHqWRhF1cgC8PUalaJ1Vnuoo7NDlfe45FnGARbqD6OVXNj1n2ed3GdTwbTaqEGkZItaXYHYGKEqc9mStACO6km7p6JmSDHLIkJkr6hI2exD6BS+CyOuAE+kChxV2iETxhk0XcSLot4fO+tNIfboUR/QqnNXKvqety8Jeew/RZDTPucG2qXmtCA94cPek= ceph-9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:58:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:57 smithi022 ceph-mon[23758]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:58:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:57 smithi022 ceph-mon[23758]: Generating ssh key... 2024-09-13T22:58:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:57 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:58:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:57 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:58:57.695 INFO:teuthology.orchestra.run.smithi022.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-13T22:58:57.695 INFO:teuthology.orchestra.run.smithi022.stderr:Adding key to root@localhost authorized_keys... 2024-09-13T22:58:57.700 INFO:teuthology.orchestra.run.smithi022.stderr:Adding host smithi022... 2024-09-13T22:58:58.136 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:58 smithi022 ceph-mon[23758]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:58:59.237 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Added host 'smithi022' with addr '172.21.15.22' 2024-09-13T22:58:59.249 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:59 smithi022 ceph-mon[23758]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi022", "addr": "172.21.15.22", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:58:59.249 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:58:59 smithi022 ceph-mon[23758]: Deploying cephadm binary to smithi022 2024-09-13T22:58:59.851 INFO:teuthology.orchestra.run.smithi022.stderr:Deploying unmanaged mon service... 2024-09-13T22:59:00.527 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:00 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:00.527 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:00 smithi022 ceph-mon[23758]: Added host smithi022 2024-09-13T22:59:00.527 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:00 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:00.527 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-13T22:59:01.259 INFO:teuthology.orchestra.run.smithi022.stderr:Deploying unmanaged mgr service... 2024-09-13T22:59:01.786 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:01 smithi022 ceph-mon[23758]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:01.786 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:01 smithi022 ceph-mon[23758]: Saving service mon spec with placement count:5 2024-09-13T22:59:01.786 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:01 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:02.020 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-13T22:59:03.353 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:03 smithi022 ceph-mon[23758]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:03.353 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:03 smithi022 ceph-mon[23758]: Saving service mgr spec with placement count:2 2024-09-13T22:59:03.354 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:03 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:03.354 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:03 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:04.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:04 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:04.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:04 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/648196599' entity='client.admin' 2024-09-13T22:59:05.959 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:05 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:06.589 INFO:teuthology.orchestra.run.smithi022.stderr:Enabling the dashboard module... 2024-09-13T22:59:07.008 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:06 smithi022 ceph-mon[23758]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:07.008 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:06 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3559726963' entity='client.admin' 2024-09-13T22:59:08.390 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:08 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/361269996' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T22:59:08.391 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:08 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ignoring --setuser ceph since I am not root 2024-09-13T22:59:08.391 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:08 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ignoring --setgroup ceph since I am not root 2024-09-13T22:59:08.391 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:08 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:08.388+0000 7fa560d63000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:59:08.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:08 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:08.471+0000 7fa560d63000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:59:08.993 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:08 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:08.679+0000 7fa560d63000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:59:08.993 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:08 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:08.772+0000 7fa560d63000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:59:08.993 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:08 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:08.915+0000 7fa560d63000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:59:09.277 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:09 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:09.137+0000 7fa560d63000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:59:09.277 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:09 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/361269996' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T22:59:09.277 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:09 smithi022 ceph-mon[23758]: mgrmap e9: y(active, since 17s) 2024-09-13T22:59:09.584 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:09 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:09.492+0000 7fa560d63000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:59:09.585 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:09 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:09.583+0000 7fa560d63000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:59:09.624 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-09-13T22:59:09.624 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-13T22:59:09.624 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "available": true, 2024-09-13T22:59:09.625 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "active_name": "y", 2024-09-13T22:59:09.625 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-13T22:59:09.625 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-09-13T22:59:09.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:09 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:09.759+0000 7fa560d63000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:59:10.297 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:10 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/523060946' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T22:59:10.300 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for the mgr to restart... 2024-09-13T22:59:10.300 INFO:teuthology.orchestra.run.smithi022.stderr:Waiting for mgr epoch 9... 2024-09-13T22:59:10.968 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:10 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:10.873+0000 7fa560d63000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:59:10.969 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:10 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:10.967+0000 7fa560d63000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:59:12.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:11.903+0000 7fa560d63000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:59:12.553 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:12.220+0000 7fa560d63000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:59:12.554 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:12.358+0000 7fa560d63000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:59:12.554 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:12.450+0000 7fa560d63000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:59:12.554 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:12.552+0000 7fa560d63000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:59:12.892 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:12.642+0000 7fa560d63000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:59:12.893 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:12.795+0000 7fa560d63000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:59:12.893 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:12.891+0000 7fa560d63000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:59:13.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:12.990+0000 7fa560d63000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:59:13.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:13.284+0000 7fa560d63000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:59:13.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:13.374+0000 7fa560d63000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:59:13.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:13.465+0000 7fa560d63000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:59:14.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:14 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T22:59:14.571+0000 7fa560d63000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:59:15.426 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:22:59:15] ENGINE Bus STARTING 2024-09-13T22:59:15.426 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:22:59:15] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T22:59:15.426 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:22:59:15] ENGINE Bus STARTED 2024-09-13T22:59:15.636 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: { 2024-09-13T22:59:15.636 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-13T22:59:15.637 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: "initialized": true 2024-09-13T22:59:15.637 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: } 2024-09-13T22:59:15.826 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: Active manager daemon y restarted 2024-09-13T22:59:15.826 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: Activating manager daemon y 2024-09-13T22:59:15.826 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: osdmap e3: 0 total, 0 up, 0 in 2024-09-13T22:59:15.826 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: mgrmap e10: y(active, starting, since 0.0527947s) 2024-09-13T22:59:15.826 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:59:15.826 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:59:15.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:59:15.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:59:15.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:59:15.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: Manager daemon y is now available 2024-09-13T22:59:15.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:15.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:15.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:59:15.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:59:15.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: [13/Sep/2024:22:59:15] ENGINE Bus STARTING 2024-09-13T22:59:15.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: [13/Sep/2024:22:59:15] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T22:59:15.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: [13/Sep/2024:22:59:15] ENGINE Bus STARTED 2024-09-13T22:59:15.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:16.454 INFO:teuthology.orchestra.run.smithi022.stderr:mgr epoch 9 is available 2024-09-13T22:59:16.454 INFO:teuthology.orchestra.run.smithi022.stderr:Generating a dashboard self-signed certificate... 2024-09-13T22:59:16.718 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:16 smithi022 ceph-mon[23758]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T22:59:16.718 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:16 smithi022 ceph-mon[23758]: mgrmap e11: y(active, since 1.05521s) 2024-09-13T22:59:16.719 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:16 smithi022 ceph-mon[23758]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T22:59:16.719 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:16 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:17.304 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-13T22:59:18.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:18 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:18.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:18 smithi022 ceph-mon[23758]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:18.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:18 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:18.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:18 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:18.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:18 smithi022 ceph-mon[23758]: mgrmap e12: y(active, since 2s) 2024-09-13T22:59:19.109 INFO:teuthology.orchestra.run.smithi022.stderr:Creating initial admin user... 2024-09-13T22:59:20.043 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$QZwMQYxavHkyXf0iuA2QG.Ajvoaw3am.z6z40pcrVKvaB1L0MV/VC", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726268360, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-13T22:59:20.666 INFO:teuthology.orchestra.run.smithi022.stderr:Fetching dashboard port number... 2024-09-13T22:59:21.362 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:21 smithi022 ceph-mon[23758]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:21.362 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:21 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:21.363 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: 8443 2024-09-13T22:59:21.948 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: disabled 2024-09-13T22:59:21.955 INFO:teuthology.orchestra.run.smithi022.stderr:systemctl: inactive 2024-09-13T22:59:21.956 INFO:teuthology.orchestra.run.smithi022.stderr:firewalld.service is not enabled 2024-09-13T22:59:21.956 INFO:teuthology.orchestra.run.smithi022.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-13T22:59:21.958 INFO:teuthology.orchestra.run.smithi022.stderr:Ceph Dashboard is now available at: 2024-09-13T22:59:21.958 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-09-13T22:59:21.958 INFO:teuthology.orchestra.run.smithi022.stderr: URL: https://smithi022.front.sepia.ceph.com:8443/ 2024-09-13T22:59:21.959 INFO:teuthology.orchestra.run.smithi022.stderr: User: admin 2024-09-13T22:59:21.959 INFO:teuthology.orchestra.run.smithi022.stderr: Password: nvl6yrraqy 2024-09-13T22:59:21.959 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-09-13T22:59:21.959 INFO:teuthology.orchestra.run.smithi022.stderr:Enabling autotune for osd_memory_target 2024-09-13T22:59:22.271 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:22 smithi022 ceph-mon[23758]: mgrmap e13: y(active, since 6s) 2024-09-13T22:59:22.272 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:22 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3702107668' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T22:59:24.036 INFO:teuthology.orchestra.run.smithi022.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-13T22:59:24.584 INFO:teuthology.orchestra.run.smithi022.stderr:You can access the Ceph CLI with: 2024-09-13T22:59:24.584 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-09-13T22:59:24.584 INFO:teuthology.orchestra.run.smithi022.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:59:24.584 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-09-13T22:59:24.584 INFO:teuthology.orchestra.run.smithi022.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-13T22:59:24.584 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-09-13T22:59:24.584 INFO:teuthology.orchestra.run.smithi022.stderr: ceph telemetry on 2024-09-13T22:59:24.584 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-09-13T22:59:24.584 INFO:teuthology.orchestra.run.smithi022.stderr:For more information see: 2024-09-13T22:59:24.584 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-09-13T22:59:24.585 INFO:teuthology.orchestra.run.smithi022.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-13T22:59:24.585 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-09-13T22:59:24.585 INFO:teuthology.orchestra.run.smithi022.stderr:Bootstrap complete. 2024-09-13T22:59:24.623 INFO:tasks.cephadm:Fetching config... 2024-09-13T22:59:24.624 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-13T22:59:24.624 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-13T22:59:24.642 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-13T22:59:24.642 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-13T22:59:24.642 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-13T22:59:24.698 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-13T22:59:24.699 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-13T22:59:24.699 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-13T22:59:24.766 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-13T22:59:24.767 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-13T22:59:24.767 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-13T22:59:24.824 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-13T22:59:24.824 DEBUG:teuthology.orchestra.run.smithi022:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSH86XeZiumi+4hcnigksZzO/3+0CcN35SIxtAGMjFu5o/S1doQPBcXn64HjGfBj6tGKowMu40nTmy3LMN250KUxN2D3vNpLWiu7eY2ANP1EKts+qKWaHe4iEvuT2CJo+5N0etFn7tZf63JctFXHndRdcBjk3ZJ5dIRrEcwk+1MgNJnIm8DTiOUKtTrM0m+GkzRYt6FwzW5h837gWIu7vgxiEQawFxhn1PaMQb69nURI9zD9FXDnRLBPfOs5PDtpsQmMd0/t2PO9fqMrvFXUyXdMqIvue35W0Wri2ogF7tsCR1Z1cs57Maf8hRKgd3Eh66hogTHqWRhF1cgC8PUalaJ1Vnuoo7NDlfe45FnGARbqD6OVXNj1n2ed3GdTwbTaqEGkZItaXYHYGKEqc9mStACO6km7p6JmSDHLIkJkr6hI2exD6BS+CyOuAE+kChxV2iETxhk0XcSLot4fO+tNIfboUR/QqnNXKvqety8Jeew/RZDTPucG2qXmtCA94cPek= ceph-9adcac8c-7223-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T22:59:24.909 INFO:teuthology.orchestra.run.smithi022.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSH86XeZiumi+4hcnigksZzO/3+0CcN35SIxtAGMjFu5o/S1doQPBcXn64HjGfBj6tGKowMu40nTmy3LMN250KUxN2D3vNpLWiu7eY2ANP1EKts+qKWaHe4iEvuT2CJo+5N0etFn7tZf63JctFXHndRdcBjk3ZJ5dIRrEcwk+1MgNJnIm8DTiOUKtTrM0m+GkzRYt6FwzW5h837gWIu7vgxiEQawFxhn1PaMQb69nURI9zD9FXDnRLBPfOs5PDtpsQmMd0/t2PO9fqMrvFXUyXdMqIvue35W0Wri2ogF7tsCR1Z1cs57Maf8hRKgd3Eh66hogTHqWRhF1cgC8PUalaJ1Vnuoo7NDlfe45FnGARbqD6OVXNj1n2ed3GdTwbTaqEGkZItaXYHYGKEqc9mStACO6km7p6JmSDHLIkJkr6hI2exD6BS+CyOuAE+kChxV2iETxhk0XcSLot4fO+tNIfboUR/QqnNXKvqety8Jeew/RZDTPucG2qXmtCA94cPek= ceph-9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:59:24.922 DEBUG:teuthology.orchestra.run.smithi076:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSH86XeZiumi+4hcnigksZzO/3+0CcN35SIxtAGMjFu5o/S1doQPBcXn64HjGfBj6tGKowMu40nTmy3LMN250KUxN2D3vNpLWiu7eY2ANP1EKts+qKWaHe4iEvuT2CJo+5N0etFn7tZf63JctFXHndRdcBjk3ZJ5dIRrEcwk+1MgNJnIm8DTiOUKtTrM0m+GkzRYt6FwzW5h837gWIu7vgxiEQawFxhn1PaMQb69nURI9zD9FXDnRLBPfOs5PDtpsQmMd0/t2PO9fqMrvFXUyXdMqIvue35W0Wri2ogF7tsCR1Z1cs57Maf8hRKgd3Eh66hogTHqWRhF1cgC8PUalaJ1Vnuoo7NDlfe45FnGARbqD6OVXNj1n2ed3GdTwbTaqEGkZItaXYHYGKEqc9mStACO6km7p6JmSDHLIkJkr6hI2exD6BS+CyOuAE+kChxV2iETxhk0XcSLot4fO+tNIfboUR/QqnNXKvqety8Jeew/RZDTPucG2qXmtCA94cPek= ceph-9adcac8c-7223-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T22:59:25.044 INFO:teuthology.orchestra.run.smithi076.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSH86XeZiumi+4hcnigksZzO/3+0CcN35SIxtAGMjFu5o/S1doQPBcXn64HjGfBj6tGKowMu40nTmy3LMN250KUxN2D3vNpLWiu7eY2ANP1EKts+qKWaHe4iEvuT2CJo+5N0etFn7tZf63JctFXHndRdcBjk3ZJ5dIRrEcwk+1MgNJnIm8DTiOUKtTrM0m+GkzRYt6FwzW5h837gWIu7vgxiEQawFxhn1PaMQb69nURI9zD9FXDnRLBPfOs5PDtpsQmMd0/t2PO9fqMrvFXUyXdMqIvue35W0Wri2ogF7tsCR1Z1cs57Maf8hRKgd3Eh66hogTHqWRhF1cgC8PUalaJ1Vnuoo7NDlfe45FnGARbqD6OVXNj1n2ed3GdTwbTaqEGkZItaXYHYGKEqc9mStACO6km7p6JmSDHLIkJkr6hI2exD6BS+CyOuAE+kChxV2iETxhk0XcSLot4fO+tNIfboUR/QqnNXKvqety8Jeew/RZDTPucG2qXmtCA94cPek= ceph-9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:59:25.058 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-13T22:59:25.366 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:25 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3966515712' entity='client.admin' 2024-09-13T22:59:28.561 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-13T22:59:28.561 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-13T22:59:28.745 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:28 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2097002482' entity='client.admin' 2024-09-13T22:59:30.416 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:30 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:30.416 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:30 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi022", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:59:30.416 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:30 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:30.417 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:30 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:30.417 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:30 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:31.820 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi076 2024-09-13T22:59:31.821 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-13T22:59:31.821 DEBUG:teuthology.orchestra.run.smithi076:> dd of=/etc/ceph/ceph.conf 2024-09-13T22:59:31.839 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-13T22:59:31.840 DEBUG:teuthology.orchestra.run.smithi076:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:59:31.898 INFO:tasks.cephadm:Adding host smithi076 to orchestrator... 2024-09-13T22:59:31.898 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch host add smithi076 2024-09-13T22:59:32.399 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:32 smithi022 ceph-mon[23758]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:32.399 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:32 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:32.399 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:32 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:32.399 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:32 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:59:32.399 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:32 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:59:32.399 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:32 smithi022 ceph-mon[23758]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T22:59:32.399 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:32 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:32.399 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:32 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:33.372 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:33 smithi022 ceph-mon[23758]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:59:34.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:34 smithi022 ceph-mon[23758]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi076", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:34.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:34 smithi022 ceph-mon[23758]: Deploying cephadm binary to smithi076 2024-09-13T22:59:34.699 INFO:teuthology.orchestra.run.smithi022.stdout:Added host 'smithi076' with addr '172.21.15.76' 2024-09-13T22:59:35.318 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-13T22:59:35.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:35 smithi022 ceph-mon[23758]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:35.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:35.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:35 smithi022 ceph-mon[23758]: Added host smithi076 2024-09-13T22:59:35.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:35.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:59:35.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:59:35.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:35.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:37.234 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T22:59:37.234 INFO:teuthology.orchestra.run.smithi022.stdout:[{"addr": "172.21.15.22", "hostname": "smithi022", "labels": [], "status": ""}, {"addr": "172.21.15.76", "hostname": "smithi076", "labels": [], "status": ""}] 2024-09-13T22:59:37.637 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:37 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:37.637 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:37 smithi022 ceph-mon[23758]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:37.637 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:37 smithi022 ceph-mon[23758]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:59:37.849 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-13T22:59:37.849 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-13T22:59:40.096 INFO:teuthology.orchestra.run.smithi022.stderr:adjusted tunables profile to default 2024-09-13T22:59:40.408 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:40 smithi022 ceph-mon[23758]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:40.408 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:40 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:40.408 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:40 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:40.408 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:40 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi076", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:59:40.408 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:40 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:40.408 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:40 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1207282331' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T22:59:40.705 INFO:tasks.cephadm:Adding mon.a on smithi022 2024-09-13T22:59:40.705 INFO:tasks.cephadm:Adding mon.c on smithi022 2024-09-13T22:59:40.705 INFO:tasks.cephadm:Adding mon.b on smithi076 2024-09-13T22:59:40.705 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi076:172.21.15.76=b' 2024-09-13T22:59:41.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:41 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1207282331' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T22:59:41.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:41 smithi022 ceph-mon[23758]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T22:59:42.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:42 smithi022 ceph-mon[23758]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:42.727 INFO:teuthology.orchestra.run.smithi076.stdout:Scheduled mon update... 2024-09-13T22:59:43.417 DEBUG:teuthology.orchestra.run.smithi022:mon.c> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.c.service 2024-09-13T22:59:43.419 DEBUG:teuthology.orchestra.run.smithi076:mon.b> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.b.service 2024-09-13T22:59:43.422 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T22:59:43.422 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:59:44.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:44.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi076:172.21.15.76=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:44.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: Saving service mon spec with placement smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi076:172.21.15.76=b;count:3 2024-09-13T22:59:44.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:44.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:44.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:59:44.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:59:44.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T22:59:44.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:59:44.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:59:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:59:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:43 smithi022 ceph-mon[23758]: Deploying daemon mon.c on smithi022 2024-09-13T22:59:45.349 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T22:59:45.350 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":1,"fsid":"9adcac8c-7223-11ef-bceb-c7b262605968","modified":"2024-09-13T22:58:24.106959Z","created":"2024-09-13T22:58:24.106959Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T22:59:45.351 INFO:teuthology.orchestra.run.smithi076.stderr:dumped monmap epoch 1 2024-09-13T22:59:45.900 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:45 smithi022 ceph-mon[23758]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:45.901 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:45 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/2192902821' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 podman[28421]: 2024-09-13 22:59:46.018126731 +0000 UTC m=+0.016704324 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T22:59:46.482 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 podman[28421]: 2024-09-13 22:59:46.365636278 +0000 UTC m=+0.364213877 container init 336a6f00a2a741e6666e2d455962c0dc9270df596d3344eb03571323923cea58 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c, io.buildah.version=1.19.8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_CLEAN=True, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, ceph=True, version=8, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, architecture=x86_64, io.openshift.expose-services=, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-13T22:59:46.482 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 podman[28421]: 2024-09-13 22:59:46.369379143 +0000 UTC m=+0.367956733 container start 336a6f00a2a741e6666e2d455962c0dc9270df596d3344eb03571323923cea58 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , version=8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_BRANCH=HEAD, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-13T22:59:46.483 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T22:59:46.483 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T22:59:46.483 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: pidfile_write: ignore empty --pid-file 2024-09-13T22:59:46.483 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: load: jerasure load: lrc 2024-09-13T22:59:46.485 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: RocksDB version: 6.15.5 2024-09-13T22:59:46.485 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T22:59:46.485 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Compile date Apr 18 2022 2024-09-13T22:59:46.485 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: DB SUMMARY 2024-09-13T22:59:46.485 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: DB Session ID: 6AWZZCJT4WQAY2008CR5 2024-09-13T22:59:46.485 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: CURRENT file: CURRENT 2024-09-13T22:59:46.486 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T22:59:46.486 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T22:59:46.486 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-13T22:59:46.486 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-09-13T22:59:46.486 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.error_if_exists: 0 2024-09-13T22:59:46.487 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.create_if_missing: 0 2024-09-13T22:59:46.487 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.paranoid_checks: 1 2024-09-13T22:59:46.487 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T22:59:46.487 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.env: 0x55be184c6860 2024-09-13T22:59:46.487 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.fs: Posix File System 2024-09-13T22:59:46.488 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.info_log: 0x55be19e6fdc0 2024-09-13T22:59:46.488 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T22:59:46.488 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.statistics: (nil) 2024-09-13T22:59:46.488 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.use_fsync: 0 2024-09-13T22:59:46.488 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_log_file_size: 0 2024-09-13T22:59:46.488 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T22:59:46.489 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T22:59:46.489 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T22:59:46.489 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T22:59:46.489 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.allow_fallocate: 1 2024-09-13T22:59:46.489 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T22:59:46.489 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T22:59:46.489 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.use_direct_reads: 0 2024-09-13T22:59:46.490 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T22:59:46.490 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T22:59:46.490 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.db_log_dir: 2024-09-13T22:59:46.490 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-13T22:59:46.490 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T22:59:46.490 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T22:59:46.491 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T22:59:46.491 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T22:59:46.491 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T22:59:46.491 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T22:59:46.491 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T22:59:46.491 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T22:59:46.491 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.write_buffer_manager: 0x55be19f602d0 2024-09-13T22:59:46.491 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T22:59:46.491 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T22:59:46.491 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T22:59:46.492 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T22:59:46.492 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T22:59:46.492 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T22:59:46.492 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T22:59:46.492 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T22:59:46.492 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T22:59:46.492 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.unordered_write: 0 2024-09-13T22:59:46.492 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T22:59:46.492 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T22:59:46.492 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T22:59:46.492 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T22:59:46.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.row_cache: None 2024-09-13T22:59:46.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.wal_filter: None 2024-09-13T22:59:46.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T22:59:46.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T22:59:46.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.preserve_deletes: 0 2024-09-13T22:59:46.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.two_write_queues: 0 2024-09-13T22:59:46.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T22:59:46.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.atomic_flush: 0 2024-09-13T22:59:46.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T22:59:46.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T22:59:46.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.log_readahead_size: 0 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_background_jobs: 2 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_background_compactions: -1 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_subcompactions: 1 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T22:59:46.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_open_files: -1 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_background_flushes: -1 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Compression algorithms supported: 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T22:59:46.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: kZSTD supported: 0 2024-09-13T22:59:46.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: kXpressCompression supported: 0 2024-09-13T22:59:46.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T22:59:46.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: kLZ4Compression supported: 1 2024-09-13T22:59:46.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: kBZip2Compression supported: 0 2024-09-13T22:59:46.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: kZlibCompression supported: 1 2024-09-13T22:59:46.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: kSnappyCompression supported: 1 2024-09-13T22:59:46.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T22:59:46.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-13T22:59:46.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T22:59:46.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T22:59:46.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.merge_operator: 2024-09-13T22:59:46.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_filter: None 2024-09-13T22:59:46.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T22:59:46.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T22:59:46.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T22:59:46.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T22:59:46.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55be19e3bd00) 2024-09-13T22:59:46.497 INFO:journalctl@ceph.mon.c.smithi022.stdout: cache_index_and_filter_blocks: 1 2024-09-13T22:59:46.497 INFO:journalctl@ceph.mon.c.smithi022.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T22:59:46.497 INFO:journalctl@ceph.mon.c.smithi022.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T22:59:46.497 INFO:journalctl@ceph.mon.c.smithi022.stdout: pin_top_level_index_and_filter: 1 2024-09-13T22:59:46.497 INFO:journalctl@ceph.mon.c.smithi022.stdout: index_type: 0 2024-09-13T22:59:46.497 INFO:journalctl@ceph.mon.c.smithi022.stdout: data_block_index_type: 0 2024-09-13T22:59:46.498 INFO:journalctl@ceph.mon.c.smithi022.stdout: index_shortening: 1 2024-09-13T22:59:46.498 INFO:journalctl@ceph.mon.c.smithi022.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T22:59:46.498 INFO:journalctl@ceph.mon.c.smithi022.stdout: hash_index_allow_collision: 1 2024-09-13T22:59:46.498 INFO:journalctl@ceph.mon.c.smithi022.stdout: checksum: 1 2024-09-13T22:59:46.498 INFO:journalctl@ceph.mon.c.smithi022.stdout: no_block_cache: 0 2024-09-13T22:59:46.498 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache: 0x55be19ea6170 2024-09-13T22:59:46.498 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache_name: BinnedLRUCache 2024-09-13T22:59:46.498 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache_options: 2024-09-13T22:59:46.498 INFO:journalctl@ceph.mon.c.smithi022.stdout: capacity : 536870912 2024-09-13T22:59:46.498 INFO:journalctl@ceph.mon.c.smithi022.stdout: num_shard_bits : 4 2024-09-13T22:59:46.498 INFO:journalctl@ceph.mon.c.smithi022.stdout: strict_capacity_limit : 0 2024-09-13T22:59:46.498 INFO:journalctl@ceph.mon.c.smithi022.stdout: high_pri_pool_ratio: 0.000 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache_compressed: (nil) 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: persistent_cache: (nil) 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_size: 4096 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_size_deviation: 10 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_restart_interval: 16 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: index_block_restart_interval: 1 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: metadata_block_size: 4096 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: partition_filters: 0 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: use_delta_encoding: 1 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: whole_key_filtering: 1 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: verify_compression: 0 2024-09-13T22:59:46.499 INFO:journalctl@ceph.mon.c.smithi022.stdout: read_amp_bytes_per_bit: 0 2024-09-13T22:59:46.500 INFO:journalctl@ceph.mon.c.smithi022.stdout: format_version: 4 2024-09-13T22:59:46.500 INFO:journalctl@ceph.mon.c.smithi022.stdout: enable_index_compression: 1 2024-09-13T22:59:46.500 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_align: 0 2024-09-13T22:59:46.500 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T22:59:46.500 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T22:59:46.500 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compression: NoCompression 2024-09-13T22:59:46.500 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T22:59:46.500 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T22:59:46.500 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T22:59:46.500 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.num_levels: 7 2024-09-13T22:59:46.501 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T22:59:46.501 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T22:59:46.501 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T22:59:46.501 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T22:59:46.501 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T22:59:46.501 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T22:59:46.501 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T22:59:46.501 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:59:46.501 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T22:59:46.501 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T22:59:46.501 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T22:59:46.502 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T22:59:46.502 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T22:59:46.502 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T22:59:46.502 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:59:46.502 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T22:59:46.502 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T22:59:46.504 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T22:59:46.504 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T22:59:46.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T22:59:46.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.table_properties_collectors: 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.inplace_update_support: 0 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.bloom_locality: 0 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.max_successive_merges: 0 2024-09-13T22:59:46.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.ttl: 2592000 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.enable_blob_files: false 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.min_blob_size: 0 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T22:59:46.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-13T22:59:46.509 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-13T22:59:46.509 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726268386396867, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T22:59:46.509 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-13T22:59:46.509 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T22:59:46.509 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726268386397171, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726268386, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "99636e42-c187-4cb6-a3de-3fcabc194a01", "db_session_id": "6AWZZCJT4WQAY2008CR5"}} 2024-09-13T22:59:46.509 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-13T22:59:46.509 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726268386397508, "job": 1, "event": "recovery_finished"} 2024-09-13T22:59:46.509 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T22:59:46.509 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55be19e8ca80 2024-09-13T22:59:46.509 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: DB pointer 0x55be19e9c000 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** DB Stats ** 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-13T22:59:46.510 INFO:journalctl@ceph.mon.c.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:59:46.511 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:59:46.512 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:59:46.512 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:59:46.512 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:59:46.512 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:59:46.512 INFO:journalctl@ceph.mon.c.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:59:46.512 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T22:59:46.512 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:59:46.512 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T22:59:46.512 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-13T22:59:46.512 INFO:journalctl@ceph.mon.c.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:59:46.512 INFO:journalctl@ceph.mon.c.smithi022.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:59:46.513 INFO:journalctl@ceph.mon.c.smithi022.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:46.513 INFO:journalctl@ceph.mon.c.smithi022.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:46.513 INFO:journalctl@ceph.mon.c.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T22:59:46.513 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T22:59:46.513 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-13T22:59:46.513 INFO:journalctl@ceph.mon.c.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:59:46.513 INFO:journalctl@ceph.mon.c.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:59:46.513 INFO:journalctl@ceph.mon.c.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:46.513 INFO:journalctl@ceph.mon.c.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:59:46.513 INFO:journalctl@ceph.mon.c.smithi022.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:59:46.513 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:59:46.513 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: using public_addrv [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: starting mon.c rank -1 at public addrs [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] at bind addrs [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(???) e0 preinit fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).mds e1 new map 2024-09-13T22:59:46.514 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-13T22:59:46.515 INFO:journalctl@ceph.mon.c.smithi022.stdout: e1 2024-09-13T22:59:46.515 INFO:journalctl@ceph.mon.c.smithi022.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T22:59:46.515 INFO:journalctl@ceph.mon.c.smithi022.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:59:46.515 INFO:journalctl@ceph.mon.c.smithi022.stdout: legacy client fscid: -1 2024-09-13T22:59:46.515 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T22:59:46.515 INFO:journalctl@ceph.mon.c.smithi022.stdout: No filesystems configured 2024-09-13T22:59:46.515 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T22:59:46.515 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T22:59:46.515 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T22:59:46.515 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T22:59:46.515 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T22:59:46.515 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T22:59:46.516 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T22:59:46.516 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:59:46.516 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:59:46.516 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:59:46.516 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mkfs 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:59:46.516 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T22:59:46.516 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T22:59:46.516 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: monmap e1: 1 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0]} 2024-09-13T22:59:46.516 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: fsmap 2024-09-13T22:59:46.516 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T22:59:46.517 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e1: no daemons active 2024-09-13T22:59:46.517 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/908736449' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T22:59:46.517 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2868679168' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T22:59:46.517 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2868679168' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T22:59:46.517 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/264916777' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:59:46.517 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T22:59:46.517 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: monmap e1: 1 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0]} 2024-09-13T22:59:46.517 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: fsmap 2024-09-13T22:59:46.517 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T22:59:46.517 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e1: no daemons active 2024-09-13T22:59:46.517 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3073145017' entity='client.admin' 2024-09-13T22:59:46.517 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1189506077' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3258845055' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Activating manager daemon y 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e2: y(active, starting, since 0.00273508s) 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Manager daemon y is now available 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' 2024-09-13T22:59:46.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14100 172.21.15.22:0/2131559752' entity='mgr.y' 2024-09-13T22:59:46.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e3: y(active, since 1.00704s) 2024-09-13T22:59:46.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1532965570' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T22:59:46.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e4: y(active, since 2s) 2024-09-13T22:59:46.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3911960426' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T22:59:46.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3911960426' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T22:59:46.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3894662652' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T22:59:46.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3894662652' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T22:59:46.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e5: y(active, since 4s) 2024-09-13T22:59:46.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1921923318' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T22:59:46.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Active manager daemon y restarted 2024-09-13T22:59:46.520 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Activating manager daemon y 2024-09-13T22:59:46.520 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: osdmap e2: 0 total, 0 up, 0 in 2024-09-13T22:59:46.520 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e6: y(active, starting, since 0.0525028s) 2024-09-13T22:59:46.520 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:59:46.520 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:59:46.520 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:59:46.520 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:59:46.520 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:59:46.520 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Manager daemon y is now available 2024-09-13T22:59:46.520 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e7: y(active, since 1.05503s) 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e8: y(active, since 2s) 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: [13/Sep/2024:22:58:53] ENGINE Bus STARTING 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:46.521 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.522 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.522 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: [13/Sep/2024:22:58:53] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T22:59:46.522 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: [13/Sep/2024:22:58:53] ENGINE Bus STARTED 2024-09-13T22:59:46.522 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.522 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.522 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:46.522 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:46.523 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Generating ssh key... 2024-09-13T22:59:46.523 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.523 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.523 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:46.523 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi022", "addr": "172.21.15.22", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:46.523 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Deploying cephadm binary to smithi022 2024-09-13T22:59:46.523 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.523 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Added host smithi022 2024-09-13T22:59:46.523 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.523 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:46.523 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Saving service mon spec with placement count:5 2024-09-13T22:59:46.523 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:46.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Saving service mgr spec with placement count:2 2024-09-13T22:59:46.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/648196599' entity='client.admin' 2024-09-13T22:59:46.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14118 172.21.15.22:0/983185158' entity='mgr.y' 2024-09-13T22:59:46.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3559726963' entity='client.admin' 2024-09-13T22:59:46.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/361269996' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T22:59:46.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/361269996' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T22:59:46.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e9: y(active, since 17s) 2024-09-13T22:59:46.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/523060946' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T22:59:46.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Active manager daemon y restarted 2024-09-13T22:59:46.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Activating manager daemon y 2024-09-13T22:59:46.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: osdmap e3: 0 total, 0 up, 0 in 2024-09-13T22:59:46.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e10: y(active, starting, since 0.0527947s) 2024-09-13T22:59:46.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:59:46.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:59:46.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:59:46.526 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:59:46.526 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:59:46.526 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Manager daemon y is now available 2024-09-13T22:59:46.526 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.526 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.526 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:59:46.526 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:59:46.526 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: [13/Sep/2024:22:59:15] ENGINE Bus STARTING 2024-09-13T22:59:46.526 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: [13/Sep/2024:22:59:15] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T22:59:46.526 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: [13/Sep/2024:22:59:15] ENGINE Bus STARTED 2024-09-13T22:59:46.526 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.527 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T22:59:46.527 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e11: y(active, since 1.05521s) 2024-09-13T22:59:46.527 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T22:59:46.527 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.527 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.527 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:46.527 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.527 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.527 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e12: y(active, since 2s) 2024-09-13T22:59:46.527 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:46.527 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.528 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mgrmap e13: y(active, since 6s) 2024-09-13T22:59:46.528 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3702107668' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T22:59:46.528 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3966515712' entity='client.admin' 2024-09-13T22:59:46.528 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2097002482' entity='client.admin' 2024-09-13T22:59:46.528 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.528 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi022", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:59:46.528 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.528 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.528 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.529 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:46.529 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.529 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.529 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:59:46.529 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:59:46.529 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T22:59:46.529 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.529 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.529 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:59:46.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi076", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:46.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Deploying cephadm binary to smithi076 2024-09-13T22:59:46.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:46.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Added host smithi076 2024-09-13T22:59:46.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:59:46.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:59:46.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi076", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1207282331' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1207282331' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi076:172.21.15.76=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:59:46.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Saving service mon spec with placement smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi076:172.21.15.76=b;count:3 2024-09-13T22:59:46.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:59:46.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:59:46.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T22:59:46.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:59:46.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:46.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:59:46.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:59:46.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: Deploying daemon mon.c on smithi022 2024-09-13T22:59:46.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:46.533 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/2192902821' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:59:46.533 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-13T22:59:46.533 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 ceph-mon[28440]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-13T22:59:46.533 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 bash[28421]: 336a6f00a2a741e6666e2d455962c0dc9270df596d3344eb03571323923cea58 2024-09-13T22:59:46.533 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:46 smithi022 systemd[1]: Started Ceph mon.c for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T22:59:46.903 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T22:59:46.903 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:59:51.766 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T22:59:51.766 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":2,"fsid":"9adcac8c-7223-11ef-bceb-c7b262605968","modified":"2024-09-13T22:59:46.417007Z","created":"2024-09-13T22:58:24.106959Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3301","nonce":0},{"type":"v1","addr":"172.21.15.22:6790","nonce":0}]},"addr":"172.21.15.22:6790/0","public_addr":"172.21.15.22:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T22:59:51.767 INFO:teuthology.orchestra.run.smithi076.stderr:dumped monmap epoch 2 2024-09-13T22:59:51.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:59:51.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: mon.a calling monitor election 2024-09-13T22:59:51.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:51.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:51.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:51.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:51.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: mon.c calling monitor election 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: monmap e2: 2 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: fsmap 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: mgrmap e13: y(active, since 36s) 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: overall HEALTH_OK 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:59:51.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:59:51.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:59:51.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: mon.a calling monitor election 2024-09-13T22:59:51.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:51.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:51.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:51.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:51.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: mon.c calling monitor election 2024-09-13T22:59:51.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:51.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:51.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:51.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:51.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:51.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T22:59:51.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: monmap e2: 2 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} 2024-09-13T22:59:51.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: fsmap 2024-09-13T22:59:51.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T22:59:51.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: mgrmap e13: y(active, since 36s) 2024-09-13T22:59:51.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: overall HEALTH_OK 2024-09-13T22:59:51.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T22:59:51.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:59:51.931 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:59:53.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:52 smithi022 ceph-mon[23758]: Deploying daemon mon.b on smithi076 2024-09-13T22:59:53.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:52 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/3929818' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:59:53.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:52 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:52 smithi022 ceph-mon[28440]: Deploying daemon mon.b on smithi076 2024-09-13T22:59:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:52 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/3929818' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:59:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:52 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:59:54.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:53 smithi022 ceph-mon[23758]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:54.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:53 smithi022 ceph-mon[28440]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:54.213 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T22:59:54.213 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:59:55.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:55 smithi076 systemd[1]: Starting Ceph mon.b for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T22:59:56.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:55 smithi022 ceph-mon[23758]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:56.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:55 smithi022 ceph-mon[28440]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:56.193 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T22:59:56.193 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":2,"fsid":"9adcac8c-7223-11ef-bceb-c7b262605968","modified":"2024-09-13T22:59:46.417007Z","created":"2024-09-13T22:58:24.106959Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3301","nonce":0},{"type":"v1","addr":"172.21.15.22:6790","nonce":0}]},"addr":"172.21.15.22:6790/0","public_addr":"172.21.15.22:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T22:59:56.194 INFO:teuthology.orchestra.run.smithi076.stderr:dumped monmap epoch 2 2024-09-13T22:59:56.529 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 podman[24552]: 2024-09-13 22:59:56.17398702 +0000 UTC m=+0.109408682 container create a1194b4b1c438b62e3c161190a0aa082f86579e346d8d623de51f08776e97c85 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vcs-type=git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, version=8, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public) 2024-09-13T22:59:56.529 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 podman[24552]: 2024-09-13 22:59:56.080631834 +0000 UTC m=+0.016053505 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T22:59:57.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 22:59:56 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/353004697' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:59:57.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 22:59:56 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/353004697' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:59:57.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 podman[24552]: 2024-09-13 22:59:56.921855193 +0000 UTC m=+0.857276860 container init a1194b4b1c438b62e3c161190a0aa082f86579e346d8d623de51f08776e97c85 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b, release=754, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, ceph=True, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream) 2024-09-13T22:59:57.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 podman[24552]: 2024-09-13 22:59:56.925225154 +0000 UTC m=+0.860646815 container start a1194b4b1c438b62e3c161190a0aa082f86579e346d8d623de51f08776e97c85 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, name=centos-stream, version=8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_CLEAN=True, architecture=x86_64, io.openshift.tags=base centos centos-stream) 2024-09-13T22:59:57.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T22:59:57.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T22:59:57.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: pidfile_write: ignore empty --pid-file 2024-09-13T22:59:57.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: load: jerasure load: lrc 2024-09-13T22:59:57.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: RocksDB version: 6.15.5 2024-09-13T22:59:57.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T22:59:57.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Compile date Apr 18 2022 2024-09-13T22:59:57.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: DB SUMMARY 2024-09-13T22:59:57.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: DB Session ID: OXZAF2GYUML3YX1H2IXC 2024-09-13T22:59:57.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: CURRENT file: CURRENT 2024-09-13T22:59:57.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T22:59:57.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.error_if_exists: 0 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.create_if_missing: 0 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.paranoid_checks: 1 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.env: 0x55a5388c6860 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.fs: Posix File System 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.info_log: 0x55a5397e1dc0 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.statistics: (nil) 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.use_fsync: 0 2024-09-13T22:59:57.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_log_file_size: 0 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.allow_fallocate: 1 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.use_direct_reads: 0 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.db_log_dir: 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-09-13T22:59:57.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.write_buffer_manager: 0x55a5398d2240 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T22:59:57.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.unordered_write: 0 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.row_cache: None 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.wal_filter: None 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T22:59:57.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T22:59:57.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.preserve_deletes: 0 2024-09-13T22:59:57.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.two_write_queues: 0 2024-09-13T22:59:57.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T22:59:57.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.atomic_flush: 0 2024-09-13T22:59:57.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T22:59:57.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T22:59:57.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T22:59:57.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.log_readahead_size: 0 2024-09-13T22:59:57.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_background_jobs: 2 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_background_compactions: -1 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_subcompactions: 1 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T22:59:57.294 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_open_files: -1 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_background_flushes: -1 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Compression algorithms supported: 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: kZSTD supported: 0 2024-09-13T22:59:57.295 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: kXpressCompression supported: 0 2024-09-13T22:59:57.296 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T22:59:57.296 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: kLZ4Compression supported: 1 2024-09-13T22:59:57.296 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: kBZip2Compression supported: 0 2024-09-13T22:59:57.296 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: kZlibCompression supported: 1 2024-09-13T22:59:57.296 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: kSnappyCompression supported: 1 2024-09-13T22:59:57.296 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T22:59:57.296 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-09-13T22:59:57.296 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T22:59:57.296 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.merge_operator: 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_filter: None 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a5397add00) 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout: cache_index_and_filter_blocks: 1 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout: pin_top_level_index_and_filter: 1 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout: index_type: 0 2024-09-13T22:59:57.297 INFO:journalctl@ceph.mon.b.smithi076.stdout: data_block_index_type: 0 2024-09-13T22:59:57.298 INFO:journalctl@ceph.mon.b.smithi076.stdout: index_shortening: 1 2024-09-13T22:59:57.298 INFO:journalctl@ceph.mon.b.smithi076.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T22:59:57.298 INFO:journalctl@ceph.mon.b.smithi076.stdout: hash_index_allow_collision: 1 2024-09-13T22:59:57.298 INFO:journalctl@ceph.mon.b.smithi076.stdout: checksum: 1 2024-09-13T22:59:57.298 INFO:journalctl@ceph.mon.b.smithi076.stdout: no_block_cache: 0 2024-09-13T22:59:57.298 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_cache: 0x55a539818170 2024-09-13T22:59:57.301 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_cache_name: BinnedLRUCache 2024-09-13T22:59:57.301 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_cache_options: 2024-09-13T22:59:57.301 INFO:journalctl@ceph.mon.b.smithi076.stdout: capacity : 536870912 2024-09-13T22:59:57.301 INFO:journalctl@ceph.mon.b.smithi076.stdout: num_shard_bits : 4 2024-09-13T22:59:57.301 INFO:journalctl@ceph.mon.b.smithi076.stdout: strict_capacity_limit : 0 2024-09-13T22:59:57.301 INFO:journalctl@ceph.mon.b.smithi076.stdout: high_pri_pool_ratio: 0.000 2024-09-13T22:59:57.301 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_cache_compressed: (nil) 2024-09-13T22:59:57.301 INFO:journalctl@ceph.mon.b.smithi076.stdout: persistent_cache: (nil) 2024-09-13T22:59:57.302 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_size: 4096 2024-09-13T22:59:57.302 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_size_deviation: 10 2024-09-13T22:59:57.302 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_restart_interval: 16 2024-09-13T22:59:57.302 INFO:journalctl@ceph.mon.b.smithi076.stdout: index_block_restart_interval: 1 2024-09-13T22:59:57.302 INFO:journalctl@ceph.mon.b.smithi076.stdout: metadata_block_size: 4096 2024-09-13T22:59:57.302 INFO:journalctl@ceph.mon.b.smithi076.stdout: partition_filters: 0 2024-09-13T22:59:57.302 INFO:journalctl@ceph.mon.b.smithi076.stdout: use_delta_encoding: 1 2024-09-13T22:59:57.302 INFO:journalctl@ceph.mon.b.smithi076.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout: whole_key_filtering: 1 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout: verify_compression: 0 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout: read_amp_bytes_per_bit: 0 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout: format_version: 4 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout: enable_index_compression: 1 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_align: 0 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compression: NoCompression 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T22:59:57.303 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.num_levels: 7 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T22:59:57.304 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T22:59:57.305 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T22:59:57.306 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.table_properties_collectors: 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.inplace_update_support: 0 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T22:59:57.307 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.bloom_locality: 0 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.max_successive_merges: 0 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.ttl: 2592000 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.enable_blob_files: false 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.min_blob_size: 0 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T22:59:57.308 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726268396952180, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726268396952486, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726268396, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "f2650d19-b4a6-4dd8-b63c-c06c19f7227d", "db_session_id": "OXZAF2GYUML3YX1H2IXC"}} 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726268396952786, "job": 1, "event": "recovery_finished"} 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55a5397fea80 2024-09-13T22:59:57.309 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: DB pointer 0x55a53980e000 2024-09-13T22:59:57.310 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T22:59:57.310 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T22:59:57.310 INFO:journalctl@ceph.mon.b.smithi076.stdout: ** DB Stats ** 2024-09-13T22:59:57.310 INFO:journalctl@ceph.mon.b.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:59:57.310 INFO:journalctl@ceph.mon.b.smithi076.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T22:59:57.310 INFO:journalctl@ceph.mon.b.smithi076.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:59:57.310 INFO:journalctl@ceph.mon.b.smithi076.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:59:57.310 INFO:journalctl@ceph.mon.b.smithi076.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T22:59:57.310 INFO:journalctl@ceph.mon.b.smithi076.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T22:59:57.310 INFO:journalctl@ceph.mon.b.smithi076.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:59:57.310 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T22:59:57.310 INFO:journalctl@ceph.mon.b.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:59:57.311 INFO:journalctl@ceph.mon.b.smithi076.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:59:57.312 INFO:journalctl@ceph.mon.b.smithi076.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:59:57.312 INFO:journalctl@ceph.mon.b.smithi076.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:59:57.312 INFO:journalctl@ceph.mon.b.smithi076.stdout: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:59:57.312 INFO:journalctl@ceph.mon.b.smithi076.stdout: Interval compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:59:57.312 INFO:journalctl@ceph.mon.b.smithi076.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:59:57.312 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T22:59:57.312 INFO:journalctl@ceph.mon.b.smithi076.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:59:57.312 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T22:59:57.312 INFO:journalctl@ceph.mon.b.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-13T22:59:57.312 INFO:journalctl@ceph.mon.b.smithi076.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:59:57.312 INFO:journalctl@ceph.mon.b.smithi076.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:59:57.312 INFO:journalctl@ceph.mon.b.smithi076.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:59:57.313 INFO:journalctl@ceph.mon.b.smithi076.stdout: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: using public_addr v2:172.21.15.76:0/0 -> [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: starting mon.b rank -1 at public addrs [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] at bind addrs [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(???) e0 preinit fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).mds e1 new map 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).mds e1 print_map 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout: e1 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout: legacy client fscid: -1 2024-09-13T22:59:57.314 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout: No filesystems configured 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:57.315 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:59:57.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: from='client.? 172.21.15.76:0/353004697' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:59:57.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing).paxosservice(auth 1..4) refresh upgraded, format 0 -> 3 2024-09-13T22:59:57.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 ceph-mon[24587]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-13T22:59:57.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 bash[24552]: a1194b4b1c438b62e3c161190a0aa082f86579e346d8d623de51f08776e97c85 2024-09-13T22:59:57.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 22:59:56 smithi076 systemd[1]: Started Ceph mon.b for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T22:59:57.793 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T22:59:57.793 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T23:00:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:00:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: mon.c calling monitor election 2024-09-13T23:00:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: mon.a calling monitor election 2024-09-13T23:00:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:00:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: monmap e3: 3 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: fsmap 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: mgrmap e13: y(active, since 47s) 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-09-13T23:00:02.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: mon.b (rank 2) addr [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] is down (out of quorum) 2024-09-13T23:00:02.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:02.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:02.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:02.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:02.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:02.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:00:02.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: mon.c calling monitor election 2024-09-13T23:00:02.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: mon.a calling monitor election 2024-09-13T23:00:02.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:02.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:00:02.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:02.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:02.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:02.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:02.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:02.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:02.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:02.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T23:00:02.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: monmap e3: 3 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} 2024-09-13T23:00:02.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: fsmap 2024-09-13T23:00:02.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T23:00:02.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: mgrmap e13: y(active, since 47s) 2024-09-13T23:00:02.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-09-13T23:00:02.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-09-13T23:00:02.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-09-13T23:00:02.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: mon.b (rank 2) addr [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] is down (out of quorum) 2024-09-13T23:00:02.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:02.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:02.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:02.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:02.613 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T23:00:02.613 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":3,"fsid":"9adcac8c-7223-11ef-bceb-c7b262605968","modified":"2024-09-13T22:59:56.973706Z","created":"2024-09-13T22:58:24.106959Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3301","nonce":0},{"type":"v1","addr":"172.21.15.22:6790","nonce":0}]},"addr":"172.21.15.22:6790/0","public_addr":"172.21.15.22:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:3300","nonce":0},{"type":"v1","addr":"172.21.15.76:6789","nonce":0}]},"addr":"172.21.15.76:6789/0","public_addr":"172.21.15.76:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T23:00:02.615 INFO:teuthology.orchestra.run.smithi076.stderr:dumped monmap epoch 3 2024-09-13T23:00:03.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:03 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/2352099326' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T23:00:03.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:03 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:03.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:03 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/2352099326' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T23:00:03.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:03 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:03.501 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-13T23:00:03.501 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-13T23:00:04.279 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:04.279 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:00:04.279 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: mon.c calling monitor election 2024-09-13T23:00:04.279 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: mon.a calling monitor election 2024-09-13T23:00:04.279 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:04.279 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:00:04.279 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:04.279 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:04.279 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:04.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:04.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:04.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:04.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:04.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T23:00:04.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: monmap e3: 3 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} 2024-09-13T23:00:04.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: fsmap 2024-09-13T23:00:04.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T23:00:04.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: mgrmap e13: y(active, since 47s) 2024-09-13T23:00:04.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-09-13T23:00:04.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-09-13T23:00:04.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-09-13T23:00:04.281 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: mon.b (rank 2) addr [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] is down (out of quorum) 2024-09-13T23:00:04.281 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:04.281 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:04.281 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:04.281 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:04.281 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='client.? 172.21.15.76:0/2352099326' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T23:00:04.281 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: mon.b calling monitor election 2024-09-13T23:00:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T23:00:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: Reconfiguring daemon mon.a on smithi022 2024-09-13T23:00:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: mon.b calling monitor election 2024-09-13T23:00:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: mon.c calling monitor election 2024-09-13T23:00:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: mon.a calling monitor election 2024-09-13T23:00:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T23:00:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: monmap e3: 3 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} 2024-09-13T23:00:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: fsmap 2024-09-13T23:00:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T23:00:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: mgrmap e13: y(active, since 49s) 2024-09-13T23:00:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-09-13T23:00:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: Cluster is now healthy 2024-09-13T23:00:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:04 smithi076 ceph-mon[24587]: overall HEALTH_OK 2024-09-13T23:00:04.591 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: mon.b calling monitor election 2024-09-13T23:00:04.591 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T23:00:04.591 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: Reconfiguring daemon mon.a on smithi022 2024-09-13T23:00:04.591 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: mon.b calling monitor election 2024-09-13T23:00:04.591 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: mon.c calling monitor election 2024-09-13T23:00:04.591 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: mon.a calling monitor election 2024-09-13T23:00:04.592 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T23:00:04.592 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: monmap e3: 3 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} 2024-09-13T23:00:04.592 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: fsmap 2024-09-13T23:00:04.592 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T23:00:04.592 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: mgrmap e13: y(active, since 49s) 2024-09-13T23:00:04.592 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-09-13T23:00:04.592 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: Cluster is now healthy 2024-09-13T23:00:04.592 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[23758]: overall HEALTH_OK 2024-09-13T23:00:04.592 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: mon.b calling monitor election 2024-09-13T23:00:04.592 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T23:00:04.593 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: Reconfiguring daemon mon.a on smithi022 2024-09-13T23:00:04.593 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: mon.b calling monitor election 2024-09-13T23:00:04.593 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: mon.c calling monitor election 2024-09-13T23:00:04.593 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: mon.a calling monitor election 2024-09-13T23:00:04.593 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T23:00:04.593 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: monmap e3: 3 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} 2024-09-13T23:00:04.593 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: fsmap 2024-09-13T23:00:04.593 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T23:00:04.593 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: mgrmap e13: y(active, since 49s) 2024-09-13T23:00:04.593 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-09-13T23:00:04.593 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: Cluster is now healthy 2024-09-13T23:00:04.594 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:04 smithi022 ceph-mon[28440]: overall HEALTH_OK 2024-09-13T23:00:05.872 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:05 smithi022 ceph-mon[23758]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:05.872 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:05.872 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:05.873 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:05 smithi022 ceph-mon[28440]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:05.873 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:05.873 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:06.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:05 smithi076 ceph-mon[24587]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:06.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:06.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:00:06.577 INFO:teuthology.orchestra.run.smithi022.stdout:# minimal ceph.conf for 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:00:06.577 INFO:teuthology.orchestra.run.smithi022.stdout:[global] 2024-09-13T23:00:06.577 INFO:teuthology.orchestra.run.smithi022.stdout: fsid = 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:00:06.577 INFO:teuthology.orchestra.run.smithi022.stdout: mon_host = [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] 2024-09-13T23:00:07.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:07.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[23758]: Reconfiguring mon.c (monmap changed)... 2024-09-13T23:00:07.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:00:07.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:00:07.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:07.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[23758]: Reconfiguring daemon mon.c on smithi022 2024-09-13T23:00:07.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2672183615' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[28440]: Reconfiguring mon.c (monmap changed)... 2024-09-13T23:00:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:00:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:00:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[28440]: Reconfiguring daemon mon.c on smithi022 2024-09-13T23:00:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:07 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2672183615' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:07 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:07 smithi076 ceph-mon[24587]: Reconfiguring mon.c (monmap changed)... 2024-09-13T23:00:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:07 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:00:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:07 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:00:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:07 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:07 smithi076 ceph-mon[24587]: Reconfiguring daemon mon.c on smithi022 2024-09-13T23:00:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:07 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2672183615' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:07.741 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-13T23:00:07.741 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-13T23:00:07.741 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T23:00:07.770 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-13T23:00:07.771 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:00:07.836 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-13T23:00:07.836 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T23:00:07.868 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-13T23:00:07.868 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:00:07.938 INFO:tasks.cephadm:Adding mgr.y on smithi022 2024-09-13T23:00:07.938 INFO:tasks.cephadm:Adding mgr.x on smithi076 2024-09-13T23:00:07.938 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi022=y;smithi076=x' 2024-09-13T23:00:08.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:08 smithi022 ceph-mon[23758]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:08.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:08 smithi022 ceph-mon[28440]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:08.515 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:08 smithi076 ceph-mon[24587]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:09.419 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:09 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:09.419 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:09 smithi076 ceph-mon[24587]: Reconfiguring mon.b (monmap changed)... 2024-09-13T23:00:09.419 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:09 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:00:09.419 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:09 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:00:09.419 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:09 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:09.419 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:09 smithi076 ceph-mon[24587]: Reconfiguring daemon mon.b on smithi076 2024-09-13T23:00:09.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:09 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:09.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:09 smithi022 ceph-mon[23758]: Reconfiguring mon.b (monmap changed)... 2024-09-13T23:00:09.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:09 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:00:09.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:09 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:00:09.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:09 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:09.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:09 smithi022 ceph-mon[23758]: Reconfiguring daemon mon.b on smithi076 2024-09-13T23:00:09.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:09 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:09.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:09 smithi022 ceph-mon[28440]: Reconfiguring mon.b (monmap changed)... 2024-09-13T23:00:09.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:09 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:00:09.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:09 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:00:09.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:09 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:09.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:09 smithi022 ceph-mon[28440]: Reconfiguring daemon mon.b on smithi076 2024-09-13T23:00:09.824 INFO:teuthology.orchestra.run.smithi076.stdout:Scheduled mgr update... 2024-09-13T23:00:10.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:10 smithi076 ceph-mon[24587]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:10.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:10 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:10.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:10 smithi022 ceph-mon[23758]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:10.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:10 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:10.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:10 smithi022 ceph-mon[28440]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:10.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:10 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:10.854 DEBUG:teuthology.orchestra.run.smithi076:mgr.x> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.x.service 2024-09-13T23:00:10.857 INFO:tasks.cephadm:Deploying OSDs... 2024-09-13T23:00:10.857 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-13T23:00:10.857 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T23:00:10.878 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T23:00:10.878 DEBUG:teuthology.orchestra.run.smithi022:> stat /dev/vg_nvme/lv_1 2024-09-13T23:00:10.938 INFO:teuthology.orchestra.run.smithi022.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T23:00:10.938 INFO:teuthology.orchestra.run.smithi022.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T23:00:10.938 INFO:teuthology.orchestra.run.smithi022.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-13T23:00:10.938 INFO:teuthology.orchestra.run.smithi022.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T23:00:10.938 INFO:teuthology.orchestra.run.smithi022.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T23:00:10.938 INFO:teuthology.orchestra.run.smithi022.stdout:Access: 2024-09-13 22:59:45.835245883 +0000 2024-09-13T23:00:10.938 INFO:teuthology.orchestra.run.smithi022.stdout:Modify: 2024-09-13 22:57:31.550568873 +0000 2024-09-13T23:00:10.938 INFO:teuthology.orchestra.run.smithi022.stdout:Change: 2024-09-13 22:57:31.550568873 +0000 2024-09-13T23:00:10.938 INFO:teuthology.orchestra.run.smithi022.stdout: Birth: 2024-09-13 22:57:31.550568873 +0000 2024-09-13T23:00:10.939 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T23:00:11.008 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records in 2024-09-13T23:00:11.008 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records out 2024-09-13T23:00:11.008 INFO:teuthology.orchestra.run.smithi022.stderr:512 bytes copied, 0.000124944 s, 4.1 MB/s 2024-09-13T23:00:11.009 DEBUG:teuthology.orchestra.run.smithi022:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T23:00:11.070 DEBUG:teuthology.orchestra.run.smithi022:> stat /dev/vg_nvme/lv_2 2024-09-13T23:00:11.129 INFO:teuthology.orchestra.run.smithi022.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T23:00:11.129 INFO:teuthology.orchestra.run.smithi022.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T23:00:11.129 INFO:teuthology.orchestra.run.smithi022.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-09-13T23:00:11.129 INFO:teuthology.orchestra.run.smithi022.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T23:00:11.130 INFO:teuthology.orchestra.run.smithi022.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T23:00:11.130 INFO:teuthology.orchestra.run.smithi022.stdout:Access: 2024-09-13 22:59:45.835245883 +0000 2024-09-13T23:00:11.130 INFO:teuthology.orchestra.run.smithi022.stdout:Modify: 2024-09-13 22:57:31.887563956 +0000 2024-09-13T23:00:11.130 INFO:teuthology.orchestra.run.smithi022.stdout:Change: 2024-09-13 22:57:31.887563956 +0000 2024-09-13T23:00:11.130 INFO:teuthology.orchestra.run.smithi022.stdout: Birth: 2024-09-13 22:57:31.887563956 +0000 2024-09-13T23:00:11.130 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T23:00:11.197 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records in 2024-09-13T23:00:11.198 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records out 2024-09-13T23:00:11.198 INFO:teuthology.orchestra.run.smithi022.stderr:512 bytes copied, 0.00013049 s, 3.9 MB/s 2024-09-13T23:00:11.199 DEBUG:teuthology.orchestra.run.smithi022:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T23:00:11.258 DEBUG:teuthology.orchestra.run.smithi022:> stat /dev/vg_nvme/lv_3 2024-09-13T23:00:11.315 INFO:teuthology.orchestra.run.smithi022.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T23:00:11.316 INFO:teuthology.orchestra.run.smithi022.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T23:00:11.316 INFO:teuthology.orchestra.run.smithi022.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-13T23:00:11.316 INFO:teuthology.orchestra.run.smithi022.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T23:00:11.316 INFO:teuthology.orchestra.run.smithi022.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T23:00:11.316 INFO:teuthology.orchestra.run.smithi022.stdout:Access: 2024-09-13 22:59:45.836245868 +0000 2024-09-13T23:00:11.316 INFO:teuthology.orchestra.run.smithi022.stdout:Modify: 2024-09-13 22:57:32.220559098 +0000 2024-09-13T23:00:11.316 INFO:teuthology.orchestra.run.smithi022.stdout:Change: 2024-09-13 22:57:32.220559098 +0000 2024-09-13T23:00:11.316 INFO:teuthology.orchestra.run.smithi022.stdout: Birth: 2024-09-13 22:57:32.220559098 +0000 2024-09-13T23:00:11.316 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T23:00:11.381 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[23758]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi022=y;smithi076=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:00:11.381 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[23758]: Saving service mgr spec with placement smithi022=y;smithi076=x;count:2 2024-09-13T23:00:11.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:11.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:11.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:11.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:11.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:11.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:00:11.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T23:00:11.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:00:11.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:11.382 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[28440]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi022=y;smithi076=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:00:11.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[28440]: Saving service mgr spec with placement smithi022=y;smithi076=x;count:2 2024-09-13T23:00:11.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:11.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:11.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:11.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:11.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:11.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:00:11.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T23:00:11.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:00:11.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:11 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:11.384 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records in 2024-09-13T23:00:11.384 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records out 2024-09-13T23:00:11.385 INFO:teuthology.orchestra.run.smithi022.stderr:512 bytes copied, 0.000111842 s, 4.6 MB/s 2024-09-13T23:00:11.386 DEBUG:teuthology.orchestra.run.smithi022:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T23:00:11.444 DEBUG:teuthology.orchestra.run.smithi022:> stat /dev/vg_nvme/lv_4 2024-09-13T23:00:11.501 INFO:teuthology.orchestra.run.smithi022.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T23:00:11.501 INFO:teuthology.orchestra.run.smithi022.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T23:00:11.502 INFO:teuthology.orchestra.run.smithi022.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-13T23:00:11.502 INFO:teuthology.orchestra.run.smithi022.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T23:00:11.502 INFO:teuthology.orchestra.run.smithi022.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T23:00:11.502 INFO:teuthology.orchestra.run.smithi022.stdout:Access: 2024-09-13 22:59:45.836245868 +0000 2024-09-13T23:00:11.502 INFO:teuthology.orchestra.run.smithi022.stdout:Modify: 2024-09-13 22:57:32.532554546 +0000 2024-09-13T23:00:11.502 INFO:teuthology.orchestra.run.smithi022.stdout:Change: 2024-09-13 22:57:32.532554546 +0000 2024-09-13T23:00:11.503 INFO:teuthology.orchestra.run.smithi022.stdout: Birth: 2024-09-13 22:57:32.532554546 +0000 2024-09-13T23:00:11.503 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T23:00:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:11 smithi076 ceph-mon[24587]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi022=y;smithi076=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:00:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:11 smithi076 ceph-mon[24587]: Saving service mgr spec with placement smithi022=y;smithi076=x;count:2 2024-09-13T23:00:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:11 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:11 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:11 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:11 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:11.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:11 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:11.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:11 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:00:11.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:11 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T23:00:11.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:11 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:00:11.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:11 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:11.567 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records in 2024-09-13T23:00:11.567 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records out 2024-09-13T23:00:11.567 INFO:teuthology.orchestra.run.smithi022.stderr:512 bytes copied, 0.000116749 s, 4.4 MB/s 2024-09-13T23:00:11.569 DEBUG:teuthology.orchestra.run.smithi022:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T23:00:11.628 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-13T23:00:11.629 DEBUG:teuthology.orchestra.run.smithi076:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T23:00:11.645 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T23:00:11.646 DEBUG:teuthology.orchestra.run.smithi076:> stat /dev/vg_nvme/lv_1 2024-09-13T23:00:11.702 INFO:teuthology.orchestra.run.smithi076.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T23:00:11.702 INFO:teuthology.orchestra.run.smithi076.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T23:00:11.702 INFO:teuthology.orchestra.run.smithi076.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-09-13T23:00:11.702 INFO:teuthology.orchestra.run.smithi076.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T23:00:11.703 INFO:teuthology.orchestra.run.smithi076.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T23:00:11.703 INFO:teuthology.orchestra.run.smithi076.stdout:Access: 2024-09-13 22:59:55.142990008 +0000 2024-09-13T23:00:11.703 INFO:teuthology.orchestra.run.smithi076.stdout:Modify: 2024-09-13 22:57:30.874777601 +0000 2024-09-13T23:00:11.703 INFO:teuthology.orchestra.run.smithi076.stdout:Change: 2024-09-13 22:57:30.874777601 +0000 2024-09-13T23:00:11.703 INFO:teuthology.orchestra.run.smithi076.stdout: Birth: 2024-09-13 22:57:30.874777601 +0000 2024-09-13T23:00:11.703 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T23:00:11.767 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records in 2024-09-13T23:00:11.767 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records out 2024-09-13T23:00:11.767 INFO:teuthology.orchestra.run.smithi076.stderr:512 bytes copied, 0.000187546 s, 2.7 MB/s 2024-09-13T23:00:11.768 DEBUG:teuthology.orchestra.run.smithi076:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T23:00:11.825 DEBUG:teuthology.orchestra.run.smithi076:> stat /dev/vg_nvme/lv_2 2024-09-13T23:00:11.881 INFO:teuthology.orchestra.run.smithi076.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T23:00:11.881 INFO:teuthology.orchestra.run.smithi076.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T23:00:11.881 INFO:teuthology.orchestra.run.smithi076.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-13T23:00:11.881 INFO:teuthology.orchestra.run.smithi076.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T23:00:11.881 INFO:teuthology.orchestra.run.smithi076.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T23:00:11.881 INFO:teuthology.orchestra.run.smithi076.stdout:Access: 2024-09-13 22:59:55.142990008 +0000 2024-09-13T23:00:11.882 INFO:teuthology.orchestra.run.smithi076.stdout:Modify: 2024-09-13 22:57:31.214767738 +0000 2024-09-13T23:00:11.882 INFO:teuthology.orchestra.run.smithi076.stdout:Change: 2024-09-13 22:57:31.214767738 +0000 2024-09-13T23:00:11.882 INFO:teuthology.orchestra.run.smithi076.stdout: Birth: 2024-09-13 22:57:31.214767738 +0000 2024-09-13T23:00:11.883 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T23:00:11.945 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records in 2024-09-13T23:00:11.945 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records out 2024-09-13T23:00:11.945 INFO:teuthology.orchestra.run.smithi076.stderr:512 bytes copied, 0.000223049 s, 2.3 MB/s 2024-09-13T23:00:11.947 DEBUG:teuthology.orchestra.run.smithi076:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T23:00:12.003 DEBUG:teuthology.orchestra.run.smithi076:> stat /dev/vg_nvme/lv_3 2024-09-13T23:00:12.058 INFO:teuthology.orchestra.run.smithi076.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T23:00:12.058 INFO:teuthology.orchestra.run.smithi076.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T23:00:12.059 INFO:teuthology.orchestra.run.smithi076.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-13T23:00:12.059 INFO:teuthology.orchestra.run.smithi076.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T23:00:12.059 INFO:teuthology.orchestra.run.smithi076.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T23:00:12.059 INFO:teuthology.orchestra.run.smithi076.stdout:Access: 2024-09-13 22:59:55.143989979 +0000 2024-09-13T23:00:12.059 INFO:teuthology.orchestra.run.smithi076.stdout:Modify: 2024-09-13 22:57:31.519758890 +0000 2024-09-13T23:00:12.059 INFO:teuthology.orchestra.run.smithi076.stdout:Change: 2024-09-13 22:57:31.519758890 +0000 2024-09-13T23:00:12.059 INFO:teuthology.orchestra.run.smithi076.stdout: Birth: 2024-09-13 22:57:31.519758890 +0000 2024-09-13T23:00:12.059 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T23:00:12.121 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records in 2024-09-13T23:00:12.121 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records out 2024-09-13T23:00:12.121 INFO:teuthology.orchestra.run.smithi076.stderr:512 bytes copied, 0.00021265 s, 2.4 MB/s 2024-09-13T23:00:12.122 DEBUG:teuthology.orchestra.run.smithi076:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T23:00:12.179 DEBUG:teuthology.orchestra.run.smithi076:> stat /dev/vg_nvme/lv_4 2024-09-13T23:00:12.235 INFO:teuthology.orchestra.run.smithi076.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T23:00:12.235 INFO:teuthology.orchestra.run.smithi076.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T23:00:12.235 INFO:teuthology.orchestra.run.smithi076.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-13T23:00:12.236 INFO:teuthology.orchestra.run.smithi076.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T23:00:12.236 INFO:teuthology.orchestra.run.smithi076.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T23:00:12.236 INFO:teuthology.orchestra.run.smithi076.stdout:Access: 2024-09-13 22:59:55.143989979 +0000 2024-09-13T23:00:12.236 INFO:teuthology.orchestra.run.smithi076.stdout:Modify: 2024-09-13 22:57:31.847749376 +0000 2024-09-13T23:00:12.236 INFO:teuthology.orchestra.run.smithi076.stdout:Change: 2024-09-13 22:57:31.847749376 +0000 2024-09-13T23:00:12.236 INFO:teuthology.orchestra.run.smithi076.stdout: Birth: 2024-09-13 22:57:31.847749376 +0000 2024-09-13T23:00:12.236 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T23:00:12.299 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records in 2024-09-13T23:00:12.299 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records out 2024-09-13T23:00:12.300 INFO:teuthology.orchestra.run.smithi076.stderr:512 bytes copied, 0.000198765 s, 2.6 MB/s 2024-09-13T23:00:12.300 DEBUG:teuthology.orchestra.run.smithi076:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T23:00:12.357 INFO:tasks.cephadm:Deploying osd.0 on smithi022 with /dev/vg_nvme/lv_4... 2024-09-13T23:00:12.357 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T23:00:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:12 smithi076 ceph-mon[24587]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:12 smithi076 ceph-mon[24587]: Deploying daemon mgr.x on smithi076 2024-09-13T23:00:12.673 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:12 smithi022 ceph-mon[23758]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:12.674 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:12 smithi022 ceph-mon[23758]: Deploying daemon mgr.x on smithi076 2024-09-13T23:00:12.674 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:12 smithi022 ceph-mon[28440]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:12.674 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:12 smithi022 ceph-mon[28440]: Deploying daemon mgr.x on smithi076 2024-09-13T23:00:12.927 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:12 smithi076 systemd[1]: Starting Ceph mgr.x for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:00:13.269 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:13 smithi076 podman[25734]: 2024-09-13 23:00:13.139207518 +0000 UTC m=+0.099631892 container create e3643f699b3582111e1702b9ae966899b2081cef4ccac8c15a743ae41307b934 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, release=754, maintainer=Guillaume Abrioux , name=centos-stream, ceph=True, distribution-scope=public, GIT_CLEAN=True, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., vcs-type=git, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, architecture=x86_64, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-13T23:00:13.269 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:13 smithi076 podman[25734]: 2024-09-13 23:00:13.056164405 +0000 UTC m=+0.016588769 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T23:00:13.521 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:13 smithi076 podman[25734]: 2024-09-13 23:00:13.375531074 +0000 UTC m=+0.335955438 container init e3643f699b3582111e1702b9ae966899b2081cef4ccac8c15a743ae41307b934 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, architecture=x86_64, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., vcs-type=git, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, version=8, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True) 2024-09-13T23:00:13.522 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:13 smithi076 podman[25734]: 2024-09-13 23:00:13.378461416 +0000 UTC m=+0.338885784 container start e3643f699b3582111e1702b9ae966899b2081cef4ccac8c15a743ae41307b934 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, release=754, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, ceph=True, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, name=centos-stream) 2024-09-13T23:00:13.522 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:13 smithi076 ceph-mgr[25753]: -- 172.21.15.76:0/1668960417 <== mon.2 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555ace7d2340 con 0x555ace7aa800 2024-09-13T23:00:13.522 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:13 smithi076 bash[25734]: e3643f699b3582111e1702b9ae966899b2081cef4ccac8c15a743ae41307b934 2024-09-13T23:00:13.522 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:13 smithi076 systemd[1]: Started Ceph mgr.x for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:00:13.789 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:13 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:13.530+0000 7f7e83418000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T23:00:13.790 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:13 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:13.614+0000 7f7e83418000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T23:00:14.189 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:13 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:13.815+0000 7f7e83418000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T23:00:14.189 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:13 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:13.927+0000 7f7e83418000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T23:00:14.189 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:14 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:14.063+0000 7f7e83418000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:00:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:14 smithi022 ceph-mon[23758]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:14 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:14 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:14 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:14 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:14 smithi022 ceph-mon[28440]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:14 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:14 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:14.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:14 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:14.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:14 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:14.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:14 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:14.276+0000 7f7e83418000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T23:00:14.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:14 smithi076 ceph-mon[24587]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:14.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:14 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:14.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:14 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:14.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:14 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:14.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:14 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:14.881 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:14 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:14.622+0000 7f7e83418000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T23:00:14.881 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:14 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:14.712+0000 7f7e83418000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T23:00:14.881 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:14 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:14.880+0000 7f7e83418000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T23:00:15.303 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:00:15.323 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi022:vg_nvme/lv_4 2024-09-13T23:00:15.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:15.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[28440]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:15.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:15.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[28440]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T23:00:15.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:00:15.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:00:15.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:15.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[28440]: Reconfiguring daemon mgr.y on smithi022 2024-09-13T23:00:15.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:00:15.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:00:15.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:15.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:15.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[23758]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:15.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:15.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[23758]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T23:00:15.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:00:15.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:00:15.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:15.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[23758]: Reconfiguring daemon mgr.y on smithi022 2024-09-13T23:00:15.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:00:15.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:00:15.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:15.933 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:15 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:15.934 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:15 smithi076 ceph-mon[24587]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:15.934 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:15 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:15.934 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:15 smithi076 ceph-mon[24587]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T23:00:15.934 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:15 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:00:15.934 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:15 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:00:15.934 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:15 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:15.934 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:15 smithi076 ceph-mon[24587]: Reconfiguring daemon mgr.y on smithi022 2024-09-13T23:00:15.934 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:15 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:00:15.934 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:15 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:00:15.934 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:15 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:16.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:15.933+0000 7f7e83418000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T23:00:16.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:16.024+0000 7f7e83418000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T23:00:17.247 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:16.938+0000 7f7e83418000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T23:00:17.247 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:17.247+0000 7f7e83418000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T23:00:17.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:17.379+0000 7f7e83418000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:00:17.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:17.468+0000 7f7e83418000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T23:00:17.618 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:17 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:17.618 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:17 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:17.618 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:17 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:17.618 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:17 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:17.618 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:17 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:17.619 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:17 smithi022 ceph-mon[23758]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:17.619 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:17 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:17.619 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:17 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:17.619 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:17 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:17.619 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:17 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:17.619 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:17 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:17.619 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:17 smithi022 ceph-mon[28440]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:17.900 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:17.949 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:17.949 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:17.949 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:17.950 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:17.950 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-mon[24587]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:17.950 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:17.567+0000 7f7e83418000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T23:00:17.950 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:17.657+0000 7f7e83418000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T23:00:17.950 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:17.805+0000 7f7e83418000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T23:00:17.950 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:17.899+0000 7f7e83418000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T23:00:18.280 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:17.996+0000 7f7e83418000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T23:00:18.280 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:18.279+0000 7f7e83418000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T23:00:18.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:18.368+0000 7f7e83418000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T23:00:18.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:18.457+0000 7f7e83418000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T23:00:18.882 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:18 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:00:18.882 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:18 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:00:18.882 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:18 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:18.882 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:18 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:00:18.883 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:18 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:00:18.883 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:18 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:19.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:18 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:00:19.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:18 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:00:19.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:18 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:19.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:00:19 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:00:19.530+0000 7f7e83418000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T23:00:19.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:19 smithi076 ceph-mon[24587]: from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:00:19.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:19 smithi076 ceph-mon[24587]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:19.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:19 smithi076 ceph-mon[24587]: Standby manager daemon x started 2024-09-13T23:00:19.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:19 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2901735839' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:00:19.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:19 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2901735839' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:00:19.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:19 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2901735839' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:00:19.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:19 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2901735839' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:00:19.876 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[23758]: from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:00:19.877 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[23758]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:19.877 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[23758]: Standby manager daemon x started 2024-09-13T23:00:19.877 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2901735839' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:00:19.877 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2901735839' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:00:19.877 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2901735839' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:00:19.877 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2901735839' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:00:19.877 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[28440]: from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:00:19.877 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[28440]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:19.877 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[28440]: Standby manager daemon x started 2024-09-13T23:00:19.878 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2901735839' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:00:19.878 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2901735839' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:00:19.878 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2901735839' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:00:19.878 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:19 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2901735839' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:00:20.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:20 smithi022 ceph-mon[23758]: mgrmap e14: y(active, since 65s), standbys: x 2024-09-13T23:00:20.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:20 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:00:20.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:20 smithi022 ceph-mon[28440]: mgrmap e14: y(active, since 65s), standbys: x 2024-09-13T23:00:20.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:20 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:00:21.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:20 smithi076 ceph-mon[24587]: mgrmap e14: y(active, since 65s), standbys: x 2024-09-13T23:00:21.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:20 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:00:21.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[28440]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:21.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3140900260' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf3e22af-e2e1-4b53-902a-31f06def8887"}]: dispatch 2024-09-13T23:00:21.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf3e22af-e2e1-4b53-902a-31f06def8887"}]: dispatch 2024-09-13T23:00:21.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cf3e22af-e2e1-4b53-902a-31f06def8887"}]': finished 2024-09-13T23:00:21.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[28440]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T23:00:21.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:21.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/171111965' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:00:21.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[23758]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:21.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3140900260' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf3e22af-e2e1-4b53-902a-31f06def8887"}]: dispatch 2024-09-13T23:00:21.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf3e22af-e2e1-4b53-902a-31f06def8887"}]: dispatch 2024-09-13T23:00:21.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cf3e22af-e2e1-4b53-902a-31f06def8887"}]': finished 2024-09-13T23:00:21.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[23758]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T23:00:21.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:21.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:21 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/171111965' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:00:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:21 smithi076 ceph-mon[24587]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:21 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3140900260' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf3e22af-e2e1-4b53-902a-31f06def8887"}]: dispatch 2024-09-13T23:00:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:21 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf3e22af-e2e1-4b53-902a-31f06def8887"}]: dispatch 2024-09-13T23:00:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:21 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cf3e22af-e2e1-4b53-902a-31f06def8887"}]': finished 2024-09-13T23:00:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:21 smithi076 ceph-mon[24587]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T23:00:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:21 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:21 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/171111965' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:00:24.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:23 smithi076 ceph-mon[24587]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:24.101 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:23 smithi022 ceph-mon[23758]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:24.101 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:23 smithi022 ceph-mon[28440]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:25.836 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:25 smithi022 ceph-mon[23758]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:25.836 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:25 smithi022 ceph-mon[28440]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:26.009 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:25 smithi076 ceph-mon[24587]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:27.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:27 smithi022 ceph-mon[28440]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:27.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T23:00:27.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:27.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:27 smithi022 ceph-mon[28440]: Deploying daemon osd.0 on smithi022 2024-09-13T23:00:27.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:27 smithi022 ceph-mon[23758]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:27.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T23:00:27.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:27.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:27 smithi022 ceph-mon[23758]: Deploying daemon osd.0 on smithi022 2024-09-13T23:00:28.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:27 smithi076 ceph-mon[24587]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:28.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T23:00:28.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:28.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:27 smithi076 ceph-mon[24587]: Deploying daemon osd.0 on smithi022 2024-09-13T23:00:29.866 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:29 smithi022 ceph-mon[23758]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:29.866 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:29 smithi022 ceph-mon[28440]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:30.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:29 smithi076 ceph-mon[24587]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:31.975 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:31 smithi022 ceph-mon[23758]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:31.976 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:31 smithi022 ceph-mon[28440]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:32.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:31 smithi076 ceph-mon[24587]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:33.277 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:32 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:33.277 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:32 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:33.277 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:32 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:33.277 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:32 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:33.277 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:32 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:33.277 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:32 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:33.278 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:32 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:33.278 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:32 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:33.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:32 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:32 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:32 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:32 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:34.216 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:33 smithi022 ceph-mon[23758]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:34.217 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:33 smithi022 ceph-mon[28440]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:33 smithi076 ceph-mon[24587]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:35.410 INFO:teuthology.orchestra.run.smithi022.stdout:Created osd(s) 0 on host 'smithi022' 2024-09-13T23:00:35.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:35.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[23758]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:35.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:35.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[23758]: from='osd.0 [v2:172.21.15.22:6802/473621849,v1:172.21.15.22:6803/473621849]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T23:00:35.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[23758]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T23:00:35.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:35.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:35.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:35.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:35.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:35.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[28440]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:35.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:35.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[28440]: from='osd.0 [v2:172.21.15.22:6802/473621849,v1:172.21.15.22:6803/473621849]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T23:00:35.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[28440]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T23:00:35.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:35.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:35.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:35.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:35 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:35.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:35 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:35.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:35 smithi076 ceph-mon[24587]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:35.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:35 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:35.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:35 smithi076 ceph-mon[24587]: from='osd.0 [v2:172.21.15.22:6802/473621849,v1:172.21.15.22:6803/473621849]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T23:00:35.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:35 smithi076 ceph-mon[24587]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T23:00:35.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:35 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:35.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:35 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:35.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:35 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:35.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:35 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:36.023 DEBUG:teuthology.orchestra.run.smithi022:osd.0> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T23:00:36.025 INFO:tasks.cephadm:Deploying osd.1 on smithi022 with /dev/vg_nvme/lv_3... 2024-09-13T23:00:36.026 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T23:00:37.134 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:36 smithi022 ceph-mon[28440]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T23:00:37.134 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:36 smithi022 ceph-mon[28440]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T23:00:37.134 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:36 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:37.134 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:36 smithi022 ceph-mon[28440]: from='osd.0 [v2:172.21.15.22:6802/473621849,v1:172.21.15.22:6803/473621849]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:00:37.134 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:36 smithi022 ceph-mon[28440]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:00:37.134 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:36 smithi022 ceph-mon[23758]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T23:00:37.134 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:36 smithi022 ceph-mon[23758]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T23:00:37.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:36 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:37.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:36 smithi022 ceph-mon[23758]: from='osd.0 [v2:172.21.15.22:6802/473621849,v1:172.21.15.22:6803/473621849]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:00:37.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:36 smithi022 ceph-mon[23758]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:00:37.135 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:00:36 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0[31271]: 2024-09-13T23:00:36.792+0000 7f1f6a6cc700 -1 osd.0 0 waiting for initial osdmap 2024-09-13T23:00:37.135 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:00:36 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0[31271]: 2024-09-13T23:00:36.794+0000 7f1f63860700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:00:37.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:36 smithi076 ceph-mon[24587]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T23:00:37.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:36 smithi076 ceph-mon[24587]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T23:00:37.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:36 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:37.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:36 smithi076 ceph-mon[24587]: from='osd.0 [v2:172.21.15.22:6802/473621849,v1:172.21.15.22:6803/473621849]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:00:37.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:36 smithi076 ceph-mon[24587]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:00:38.087 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:37 smithi022 ceph-mon[23758]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:38.087 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:37 smithi022 ceph-mon[23758]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-13T23:00:38.087 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:37 smithi022 ceph-mon[23758]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T23:00:38.088 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:37 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:38.088 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:37 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:38.088 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:37 smithi022 ceph-mon[28440]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:38.088 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:37 smithi022 ceph-mon[28440]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-13T23:00:38.088 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:37 smithi022 ceph-mon[28440]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T23:00:38.088 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:37 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:38.088 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:37 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:37 smithi076 ceph-mon[24587]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:37 smithi076 ceph-mon[24587]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-13T23:00:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:37 smithi076 ceph-mon[24587]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T23:00:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:37 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:37 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:39.084 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:38 smithi022 ceph-mon[23758]: purged_snaps scrub starts 2024-09-13T23:00:39.084 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:38 smithi022 ceph-mon[23758]: purged_snaps scrub ok 2024-09-13T23:00:39.084 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:38 smithi022 ceph-mon[23758]: osd.0 [v2:172.21.15.22:6802/473621849,v1:172.21.15.22:6803/473621849] boot 2024-09-13T23:00:39.084 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:38 smithi022 ceph-mon[23758]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T23:00:39.084 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:38 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:39.085 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:38 smithi022 ceph-mon[28440]: purged_snaps scrub starts 2024-09-13T23:00:39.088 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:38 smithi022 ceph-mon[28440]: purged_snaps scrub ok 2024-09-13T23:00:39.088 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:38 smithi022 ceph-mon[28440]: osd.0 [v2:172.21.15.22:6802/473621849,v1:172.21.15.22:6803/473621849] boot 2024-09-13T23:00:39.088 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:38 smithi022 ceph-mon[28440]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T23:00:39.088 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:38 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:39.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:38 smithi076 ceph-mon[24587]: purged_snaps scrub starts 2024-09-13T23:00:39.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:38 smithi076 ceph-mon[24587]: purged_snaps scrub ok 2024-09-13T23:00:39.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:38 smithi076 ceph-mon[24587]: osd.0 [v2:172.21.15.22:6802/473621849,v1:172.21.15.22:6803/473621849] boot 2024-09-13T23:00:39.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:38 smithi076 ceph-mon[24587]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T23:00:39.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:38 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:00:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:39 smithi022 ceph-mon[23758]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:39 smithi022 ceph-mon[23758]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T23:00:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:00:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:40.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:39 smithi022 ceph-mon[28440]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:40.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:39 smithi022 ceph-mon[28440]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T23:00:40.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:40.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:00:40.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:40.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:39 smithi076 ceph-mon[24587]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T23:00:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:39 smithi076 ceph-mon[24587]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T23:00:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:00:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:40.834 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:40 smithi022 ceph-mon[23758]: Detected new or changed devices on smithi022 2024-09-13T23:00:40.834 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:40 smithi022 ceph-mon[23758]: Adjusting osd_memory_target on smithi022 to 16068M 2024-09-13T23:00:40.834 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:40 smithi022 ceph-mon[28440]: Detected new or changed devices on smithi022 2024-09-13T23:00:40.834 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:40 smithi022 ceph-mon[28440]: Adjusting osd_memory_target on smithi022 to 16068M 2024-09-13T23:00:40.840 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:00:40.865 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi022:vg_nvme/lv_3 2024-09-13T23:00:41.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:40 smithi076 ceph-mon[24587]: Detected new or changed devices on smithi022 2024-09-13T23:00:41.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:40 smithi076 ceph-mon[24587]: Adjusting osd_memory_target on smithi022 to 16068M 2024-09-13T23:00:42.006 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:41 smithi022 ceph-mon[23758]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:42.007 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:41 smithi022 ceph-mon[28440]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:41 smithi076 ceph-mon[24587]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:43.046 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:42 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:00:43.046 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:42 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:00:43.046 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:42 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:43.046 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:42 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:00:43.047 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:42 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:00:43.047 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:42 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:42 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:00:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:42 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:00:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:42 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:43.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:43 smithi022 ceph-mon[23758]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:43.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:43 smithi022 ceph-mon[23758]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:00:43.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:43 smithi022 ceph-mon[28440]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:43.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:43 smithi022 ceph-mon[28440]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:00:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:43 smithi076 ceph-mon[24587]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:43 smithi076 ceph-mon[24587]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:00:46.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[23758]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/380847645' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb4c98bf-8fbc-4d9c-8141-598fb4168065"}]: dispatch 2024-09-13T23:00:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb4c98bf-8fbc-4d9c-8141-598fb4168065"}]: dispatch 2024-09-13T23:00:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb4c98bf-8fbc-4d9c-8141-598fb4168065"}]': finished 2024-09-13T23:00:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[23758]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T23:00:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:00:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/101350557' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:00:46.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[28440]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:46.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/380847645' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb4c98bf-8fbc-4d9c-8141-598fb4168065"}]: dispatch 2024-09-13T23:00:46.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb4c98bf-8fbc-4d9c-8141-598fb4168065"}]: dispatch 2024-09-13T23:00:46.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb4c98bf-8fbc-4d9c-8141-598fb4168065"}]': finished 2024-09-13T23:00:46.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[28440]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T23:00:46.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:00:46.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:45 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/101350557' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:00:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:45 smithi076 ceph-mon[24587]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:45 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/380847645' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb4c98bf-8fbc-4d9c-8141-598fb4168065"}]: dispatch 2024-09-13T23:00:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:45 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb4c98bf-8fbc-4d9c-8141-598fb4168065"}]: dispatch 2024-09-13T23:00:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:45 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb4c98bf-8fbc-4d9c-8141-598fb4168065"}]': finished 2024-09-13T23:00:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:45 smithi076 ceph-mon[24587]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T23:00:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:45 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:00:46.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:45 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/101350557' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:00:48.081 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:47 smithi022 ceph-mon[23758]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:48.082 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:47 smithi022 ceph-mon[28440]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:48.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:47 smithi076 ceph-mon[24587]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:50.045 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:49 smithi022 ceph-mon[23758]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:50.045 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:49 smithi022 ceph-mon[28440]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:49 smithi076 ceph-mon[24587]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:51.881 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:51 smithi022 ceph-mon[23758]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:51.881 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T23:00:51.881 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:51.881 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:51 smithi022 ceph-mon[23758]: Deploying daemon osd.1 on smithi022 2024-09-13T23:00:51.882 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:51 smithi022 ceph-mon[28440]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:51.882 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T23:00:51.882 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:51.882 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:51 smithi022 ceph-mon[28440]: Deploying daemon osd.1 on smithi022 2024-09-13T23:00:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:51 smithi076 ceph-mon[24587]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:51 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T23:00:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:51 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:51 smithi076 ceph-mon[24587]: Deploying daemon osd.1 on smithi022 2024-09-13T23:00:54.089 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:53 smithi076 ceph-mon[24587]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:54.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:53 smithi022 ceph-mon[23758]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:54.168 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:53 smithi022 ceph-mon[28440]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:56.094 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:55 smithi022 ceph-mon[23758]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:56.094 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:55 smithi022 ceph-mon[28440]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:56.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:55 smithi076 ceph-mon[24587]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:57.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:57 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:57.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:57 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:57.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:57 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:57.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:57 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:57.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:57 smithi022 ceph-mon[28440]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:57.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:57 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:57.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:57 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:57.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:57 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:57.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:57 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:57.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:57 smithi022 ceph-mon[23758]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:57.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:57 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:00:57.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:57 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:00:57.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:57 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:00:57.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:57 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:00:57.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:57 smithi076 ceph-mon[24587]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:00:59.748 INFO:teuthology.orchestra.run.smithi022.stdout:Created osd(s) 1 on host 'smithi022' 2024-09-13T23:01:00.071 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[23758]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:01:00.071 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:00.071 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:00.071 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[23758]: from='osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T23:01:00.071 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:00.071 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:00.071 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:00.071 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:00.072 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[28440]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:01:00.072 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:00.072 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:00.072 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[28440]: from='osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T23:01:00.072 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:00.072 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:00.072 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:00.072 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:00:59 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:59 smithi076 ceph-mon[24587]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:01:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:59 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:59 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:59 smithi076 ceph-mon[24587]: from='osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T23:01:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:59 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:59 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:59 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:00:59 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:00.341 DEBUG:teuthology.orchestra.run.smithi022:osd.1> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T23:01:00.343 INFO:tasks.cephadm:Deploying osd.2 on smithi022 with /dev/vg_nvme/lv_2... 2024-09-13T23:01:00.344 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T23:01:01.118 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:00 smithi022 ceph-mon[23758]: from='osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T23:01:01.118 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:00 smithi022 ceph-mon[23758]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T23:01:01.118 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:00 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:01:01.118 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:00 smithi022 ceph-mon[23758]: from='osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:01:01.119 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:00 smithi022 ceph-mon[28440]: from='osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T23:01:01.119 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:00 smithi022 ceph-mon[28440]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T23:01:01.119 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:00 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:01:01.119 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:00 smithi022 ceph-mon[28440]: from='osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:01:01.119 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:01:00 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1[34222]: 2024-09-13T23:01:00.863+0000 7f816828b700 -1 osd.1 0 waiting for initial osdmap 2024-09-13T23:01:01.119 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:01:00 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1[34222]: 2024-09-13T23:01:00.867+0000 7f8162421700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:01:01.284 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:00 smithi076 ceph-mon[24587]: from='osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T23:01:01.284 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:00 smithi076 ceph-mon[24587]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T23:01:01.285 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:00 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:01:01.285 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:00 smithi076 ceph-mon[24587]: from='osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:01:02.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:01 smithi022 ceph-mon[28440]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:01:02.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:01 smithi022 ceph-mon[28440]: from='osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-13T23:01:02.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:01 smithi022 ceph-mon[28440]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T23:01:02.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:01 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:01:02.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:01 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:01:02.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:01 smithi022 ceph-mon[23758]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:01:02.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:01 smithi022 ceph-mon[23758]: from='osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-13T23:01:02.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:01 smithi022 ceph-mon[23758]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T23:01:02.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:01 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:01:02.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:01 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:01:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:01 smithi076 ceph-mon[24587]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:01:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:01 smithi076 ceph-mon[24587]: from='osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-13T23:01:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:01 smithi076 ceph-mon[24587]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T23:01:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:01 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:01:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:01 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:01:03.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:02 smithi022 ceph-mon[23758]: purged_snaps scrub starts 2024-09-13T23:01:03.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:02 smithi022 ceph-mon[23758]: purged_snaps scrub ok 2024-09-13T23:01:03.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:02 smithi022 ceph-mon[23758]: osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463] boot 2024-09-13T23:01:03.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:02 smithi022 ceph-mon[23758]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T23:01:03.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:01:03.168 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:02 smithi022 ceph-mon[28440]: purged_snaps scrub starts 2024-09-13T23:01:03.168 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:02 smithi022 ceph-mon[28440]: purged_snaps scrub ok 2024-09-13T23:01:03.169 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:02 smithi022 ceph-mon[28440]: osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463] boot 2024-09-13T23:01:03.169 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:02 smithi022 ceph-mon[28440]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T23:01:03.169 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:01:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:02 smithi076 ceph-mon[24587]: purged_snaps scrub starts 2024-09-13T23:01:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:02 smithi076 ceph-mon[24587]: purged_snaps scrub ok 2024-09-13T23:01:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:02 smithi076 ceph-mon[24587]: osd.1 [v2:172.21.15.22:6810/1475707463,v1:172.21.15.22:6811/1475707463] boot 2024-09-13T23:01:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:02 smithi076 ceph-mon[24587]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T23:01:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:02 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:01:04.105 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:03 smithi022 ceph-mon[28440]: pgmap v57: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:01:04.105 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:03 smithi022 ceph-mon[28440]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T23:01:04.105 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:03 smithi022 ceph-mon[23758]: pgmap v57: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:01:04.105 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:03 smithi022 ceph-mon[23758]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T23:01:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:03 smithi076 ceph-mon[24587]: pgmap v57: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T23:01:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:03 smithi076 ceph-mon[24587]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T23:01:05.165 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[23758]: Detected new or changed devices on smithi022 2024-09-13T23:01:05.165 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:05.165 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:05.166 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:05.166 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[23758]: Adjusting osd_memory_target on smithi022 to 8034M 2024-09-13T23:01:05.166 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:05.166 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:05.166 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[28440]: Detected new or changed devices on smithi022 2024-09-13T23:01:05.166 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:05.166 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:05.166 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:05.166 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[28440]: Adjusting osd_memory_target on smithi022 to 8034M 2024-09-13T23:01:05.166 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:05.167 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:04 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:05.171 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:01:05.193 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi022:vg_nvme/lv_2 2024-09-13T23:01:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:04 smithi076 ceph-mon[24587]: Detected new or changed devices on smithi022 2024-09-13T23:01:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:04 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:04 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:04 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:04 smithi076 ceph-mon[24587]: Adjusting osd_memory_target on smithi022 to 8034M 2024-09-13T23:01:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:04 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:04 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:06.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:05 smithi022 ceph-mon[23758]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:06.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:05 smithi022 ceph-mon[28440]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:06.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:05 smithi076 ceph-mon[24587]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:08.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:07 smithi022 ceph-mon[23758]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:08.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:07 smithi022 ceph-mon[23758]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:01:08.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:07 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:01:08.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:07 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:01:08.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:07 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:07 smithi022 ceph-mon[28440]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:07 smithi022 ceph-mon[28440]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:01:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:07 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:01:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:07 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:01:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:07 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:07 smithi076 ceph-mon[24587]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:07 smithi076 ceph-mon[24587]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:01:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:07 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:01:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:07 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:01:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:07 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:10.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:09 smithi022 ceph-mon[23758]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:09 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1084422556' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e5b61686-bddb-4564-bc89-0da23cf64582"}]: dispatch 2024-09-13T23:01:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:09 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e5b61686-bddb-4564-bc89-0da23cf64582"}]: dispatch 2024-09-13T23:01:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:09 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e5b61686-bddb-4564-bc89-0da23cf64582"}]': finished 2024-09-13T23:01:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:09 smithi022 ceph-mon[23758]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T23:01:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:09 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:09 smithi022 ceph-mon[28440]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:09 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1084422556' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e5b61686-bddb-4564-bc89-0da23cf64582"}]: dispatch 2024-09-13T23:01:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:09 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e5b61686-bddb-4564-bc89-0da23cf64582"}]: dispatch 2024-09-13T23:01:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:09 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e5b61686-bddb-4564-bc89-0da23cf64582"}]': finished 2024-09-13T23:01:10.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:09 smithi022 ceph-mon[28440]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T23:01:10.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:09 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:09 smithi076 ceph-mon[24587]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:09 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1084422556' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e5b61686-bddb-4564-bc89-0da23cf64582"}]: dispatch 2024-09-13T23:01:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:09 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e5b61686-bddb-4564-bc89-0da23cf64582"}]: dispatch 2024-09-13T23:01:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:09 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e5b61686-bddb-4564-bc89-0da23cf64582"}]': finished 2024-09-13T23:01:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:09 smithi076 ceph-mon[24587]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T23:01:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:09 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:11.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:10 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1031379360' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:01:11.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:10 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1031379360' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:01:11.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:10 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1031379360' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:01:12.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:11 smithi022 ceph-mon[28440]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:12.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:11 smithi022 ceph-mon[23758]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:11 smithi076 ceph-mon[24587]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:13.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:13 smithi022 ceph-mon[23758]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:13.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:13 smithi022 ceph-mon[28440]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:14.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:13 smithi076 ceph-mon[24587]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:15.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:14 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:01:15.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:14 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:01:15.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:14 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:01:15.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:14 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:01:15.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:14 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:01:15.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:14 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:01:16.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:15 smithi022 ceph-mon[23758]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:16.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T23:01:16.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:16.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:15 smithi022 ceph-mon[28440]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:16.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:15 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T23:01:16.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:15 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:15 smithi076 ceph-mon[24587]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:15 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T23:01:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:15 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:17.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:16 smithi022 ceph-mon[23758]: Deploying daemon osd.2 on smithi022 2024-09-13T23:01:17.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:16 smithi022 ceph-mon[28440]: Deploying daemon osd.2 on smithi022 2024-09-13T23:01:17.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:16 smithi076 ceph-mon[24587]: Deploying daemon osd.2 on smithi022 2024-09-13T23:01:18.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:17 smithi022 ceph-mon[23758]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:18.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:17 smithi022 ceph-mon[28440]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:17 smithi076 ceph-mon[24587]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:20.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:19 smithi022 ceph-mon[23758]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:19 smithi022 ceph-mon[28440]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:19 smithi076 ceph-mon[24587]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:22.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:22 smithi022 ceph-mon[23758]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:22.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:22 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:22.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:22 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:22.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:22 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:22.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:22 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:22 smithi022 ceph-mon[28440]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:22 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:22 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:22 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:22.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:22 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:22 smithi076 ceph-mon[24587]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:22 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:22 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:22 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:22 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:24.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:24 smithi022 ceph-mon[23758]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:24.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:24 smithi022 ceph-mon[28440]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:24.207 INFO:teuthology.orchestra.run.smithi022.stdout:Created osd(s) 2 on host 'smithi022' 2024-09-13T23:01:24.339 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:24 smithi076 ceph-mon[24587]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:24.898 DEBUG:teuthology.orchestra.run.smithi022:osd.2> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.2.service 2024-09-13T23:01:24.901 INFO:tasks.cephadm:Deploying osd.3 on smithi022 with /dev/vg_nvme/lv_1... 2024-09-13T23:01:24.902 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T23:01:25.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:25 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:25.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:25 smithi022 ceph-mon[23758]: from='osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T23:01:25.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:25 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:25 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:25 smithi022 ceph-mon[28440]: from='osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T23:01:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:25 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:25.503 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:25 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:25.503 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:25 smithi076 ceph-mon[24587]: from='osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T23:01:25.503 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:25 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:26.279 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:26 smithi022 ceph-mon[23758]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:26.279 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:26 smithi022 ceph-mon[23758]: from='osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T23:01:26.279 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:26 smithi022 ceph-mon[23758]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T23:01:26.279 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:26 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:26.279 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:26 smithi022 ceph-mon[23758]: from='osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:01:26.280 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:26 smithi022 ceph-mon[28440]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:26.280 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:26 smithi022 ceph-mon[28440]: from='osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T23:01:26.280 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:26 smithi022 ceph-mon[28440]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T23:01:26.280 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:26 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:26.280 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:26 smithi022 ceph-mon[28440]: from='osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:01:26.280 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:01:26 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2[37137]: 2024-09-13T23:01:26.214+0000 7f9cd27b7700 -1 osd.2 0 waiting for initial osdmap 2024-09-13T23:01:26.280 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:01:26 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2[37137]: 2024-09-13T23:01:26.219+0000 7f9ccb14a700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:01:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:26 smithi076 ceph-mon[24587]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:26 smithi076 ceph-mon[24587]: from='osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T23:01:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:26 smithi076 ceph-mon[24587]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T23:01:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:26 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:26 smithi076 ceph-mon[24587]: from='osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:01:27.436 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:27 smithi022 ceph-mon[23758]: purged_snaps scrub starts 2024-09-13T23:01:27.436 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:27 smithi022 ceph-mon[23758]: purged_snaps scrub ok 2024-09-13T23:01:27.437 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:27 smithi022 ceph-mon[23758]: from='osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-13T23:01:27.437 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:27 smithi022 ceph-mon[23758]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T23:01:27.437 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:27.437 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:27.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:27 smithi022 ceph-mon[28440]: purged_snaps scrub starts 2024-09-13T23:01:27.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:27 smithi022 ceph-mon[28440]: purged_snaps scrub ok 2024-09-13T23:01:27.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:27 smithi022 ceph-mon[28440]: from='osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-13T23:01:27.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:27 smithi022 ceph-mon[28440]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T23:01:27.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:27.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:27 smithi076 ceph-mon[24587]: purged_snaps scrub starts 2024-09-13T23:01:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:27 smithi076 ceph-mon[24587]: purged_snaps scrub ok 2024-09-13T23:01:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:27 smithi076 ceph-mon[24587]: from='osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-13T23:01:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:27 smithi076 ceph-mon[24587]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T23:01:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:28.071 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:01:28.099 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi022:vg_nvme/lv_1 2024-09-13T23:01:28.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:28 smithi022 ceph-mon[28440]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:28.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:28 smithi022 ceph-mon[28440]: osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233] boot 2024-09-13T23:01:28.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:28 smithi022 ceph-mon[28440]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T23:01:28.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:28 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:28.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:28 smithi022 ceph-mon[23758]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:28.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:28 smithi022 ceph-mon[23758]: osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233] boot 2024-09-13T23:01:28.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:28 smithi022 ceph-mon[23758]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T23:01:28.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:28 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:28.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:28 smithi076 ceph-mon[24587]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T23:01:28.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:28 smithi076 ceph-mon[24587]: osd.2 [v2:172.21.15.22:6818/3250575233,v1:172.21.15.22:6819/3250575233] boot 2024-09-13T23:01:28.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:28 smithi076 ceph-mon[24587]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T23:01:28.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:28 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:01:29.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:29 smithi022 ceph-mon[23758]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T23:01:29.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:29 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T23:01:29.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:29 smithi022 ceph-mon[28440]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T23:01:29.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:29 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T23:01:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:29 smithi076 ceph-mon[24587]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T23:01:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:29 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T23:01:30.480 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:30 smithi022 ceph-mon[23758]: pgmap v76: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:30.480 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:30 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T23:01:30.480 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:30 smithi022 ceph-mon[23758]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T23:01:30.480 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:30 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T23:01:30.481 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:30 smithi022 ceph-mon[28440]: pgmap v76: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:30.481 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:30 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T23:01:30.481 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:30 smithi022 ceph-mon[28440]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T23:01:30.481 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:30 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T23:01:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:30 smithi076 ceph-mon[24587]: pgmap v76: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:30 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T23:01:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:30 smithi076 ceph-mon[24587]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T23:01:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:30 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T23:01:31.136 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:01:30 smithi022 sudo[38508]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T23:01:31.136 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:01:30 smithi022 sudo[38508]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T23:01:31.136 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:01:30 smithi022 sudo[38508]: pam_unix(sudo:session): session closed for user root 2024-09-13T23:01:31.426 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:01:31 smithi022 sudo[38557]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T23:01:31.426 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:01:31 smithi022 sudo[38557]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T23:01:31.426 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:01:31 smithi022 sudo[38557]: pam_unix(sudo:session): session closed for user root 2024-09-13T23:01:31.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:31.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:31.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T23:01:31.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T23:01:31.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:01:31.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:01:31.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:31.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:31.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:31.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:31.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:31.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:31.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:31.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:31.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:31.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:31.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T23:01:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T23:01:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:01:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:01:31.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:31.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:31.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:31.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:31.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:31.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:31.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:31.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:31.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:31.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:31.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:31.926 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:01:31 smithi022 sudo[38578]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T23:01:31.926 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:01:31 smithi022 sudo[38578]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T23:01:31.926 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:01:31 smithi022 sudo[38578]: pam_unix(sudo:session): session closed for user root 2024-09-13T23:01:32.326 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:01:32 smithi022 sudo[38585]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T23:01:32.326 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:01:32 smithi022 sudo[38585]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T23:01:32.326 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:01:32 smithi022 sudo[38585]: pam_unix(sudo:session): session closed for user root 2024-09-13T23:01:32.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[28440]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:32.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[28440]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:01:32.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[28440]: Detected new or changed devices on smithi022 2024-09-13T23:01:32.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[28440]: Adjusting osd_memory_target on smithi022 to 5356M 2024-09-13T23:01:32.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[28440]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T23:01:32.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[28440]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T23:01:32.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:32.327 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[23758]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:32.327 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[23758]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:01:32.328 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[23758]: Detected new or changed devices on smithi022 2024-09-13T23:01:32.328 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[23758]: Adjusting osd_memory_target on smithi022 to 5356M 2024-09-13T23:01:32.328 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[23758]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T23:01:32.328 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[23758]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T23:01:32.328 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:32 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:32 smithi076 ceph-mon[24587]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:32 smithi076 ceph-mon[24587]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:01:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:32 smithi076 ceph-mon[24587]: Detected new or changed devices on smithi022 2024-09-13T23:01:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:32 smithi076 ceph-mon[24587]: Adjusting osd_memory_target on smithi022 to 5356M 2024-09-13T23:01:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:32 smithi076 ceph-mon[24587]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T23:01:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:32 smithi076 ceph-mon[24587]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T23:01:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:32 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:32.926 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:01:32 smithi022 sudo[38607]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T23:01:32.926 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:01:32 smithi022 sudo[38607]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T23:01:32.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:01:32 smithi022 sudo[38607]: pam_unix(sudo:session): session closed for user root 2024-09-13T23:01:33.426 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:01:33 smithi022 sudo[38627]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T23:01:33.426 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:01:33 smithi022 sudo[38627]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T23:01:33.426 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:01:33 smithi022 sudo[38627]: pam_unix(sudo:session): session closed for user root 2024-09-13T23:01:33.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:33 smithi022 sudo[38655]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T23:01:33.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:33 smithi022 sudo[38655]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T23:01:33.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:33 smithi022 sudo[38655]: pam_unix(sudo:session): session closed for user root 2024-09-13T23:01:34.392 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:33 smithi022 sudo[38658]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T23:01:34.392 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:33 smithi022 sudo[38658]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T23:01:34.392 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:33 smithi022 sudo[38658]: pam_unix(sudo:session): session closed for user root 2024-09-13T23:01:34.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 sudo[38686]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T23:01:34.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 sudo[38686]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T23:01:34.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 sudo[38686]: pam_unix(sudo:session): session closed for user root 2024-09-13T23:01:35.095 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:35.096 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T23:01:35.096 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3639870622' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5bf14cf-4376-4c35-886e-1a6217af0b8d"}]: dispatch 2024-09-13T23:01:35.096 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T23:01:35.096 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:01:35.096 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5bf14cf-4376-4c35-886e-1a6217af0b8d"}]: dispatch 2024-09-13T23:01:35.096 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:01:35.096 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:01:35.096 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5bf14cf-4376-4c35-886e-1a6217af0b8d"}]': finished 2024-09-13T23:01:35.096 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T23:01:35.096 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:35.096 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:01:35.096 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:01:35.097 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:34 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:01:35.097 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 sudo[26099]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T23:01:35.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:35.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T23:01:35.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3639870622' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5bf14cf-4376-4c35-886e-1a6217af0b8d"}]: dispatch 2024-09-13T23:01:35.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T23:01:35.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:01:35.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5bf14cf-4376-4c35-886e-1a6217af0b8d"}]: dispatch 2024-09-13T23:01:35.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:01:35.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:01:35.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5bf14cf-4376-4c35-886e-1a6217af0b8d"}]': finished 2024-09-13T23:01:35.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T23:01:35.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:35.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:01:35.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:01:35.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 sudo[38689]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 sudo[38689]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 sudo[38689]: pam_unix(sudo:session): session closed for user root 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3639870622' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5bf14cf-4376-4c35-886e-1a6217af0b8d"}]: dispatch 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5bf14cf-4376-4c35-886e-1a6217af0b8d"}]: dispatch 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5bf14cf-4376-4c35-886e-1a6217af0b8d"}]': finished 2024-09-13T23:01:35.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T23:01:35.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:35.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:01:35.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:01:35.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:34 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:01:35.346 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 sudo[26099]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T23:01:35.346 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 sudo[26099]: pam_unix(sudo:session): session closed for user root 2024-09-13T23:01:35.346 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 sudo[26102]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T23:01:35.346 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 sudo[26102]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T23:01:35.346 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 sudo[26102]: pam_unix(sudo:session): session closed for user root 2024-09-13T23:01:36.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[23758]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T23:01:36.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[23758]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:36.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[23758]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T23:01:36.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[23758]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T23:01:36.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:01:36.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:01:36.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:01:36.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1082419376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:01:36.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[23758]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T23:01:36.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:36.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[28440]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T23:01:36.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[28440]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:36.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[28440]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T23:01:36.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[28440]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T23:01:36.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:01:36.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:01:36.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:01:36.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1082419376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:01:36.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[28440]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T23:01:36.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:35 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 ceph-mon[24587]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T23:01:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 ceph-mon[24587]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 ceph-mon[24587]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T23:01:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 ceph-mon[24587]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T23:01:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:01:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:01:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:01:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1082419376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:01:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 ceph-mon[24587]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T23:01:36.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:35 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:37.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:37 smithi022 ceph-mon[28440]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:37.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:37 smithi022 ceph-mon[23758]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:37 smithi076 ceph-mon[24587]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:39.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:39 smithi022 ceph-mon[23758]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.8 KiB/s rd, 167 KiB/s wr, 11 op/s 2024-09-13T23:01:39.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:39 smithi022 ceph-mon[28440]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.8 KiB/s rd, 167 KiB/s wr, 11 op/s 2024-09-13T23:01:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:39 smithi076 ceph-mon[24587]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.8 KiB/s rd, 167 KiB/s wr, 11 op/s 2024-09-13T23:01:41.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:41 smithi022 ceph-mon[23758]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 117 KiB/s wr, 8 op/s 2024-09-13T23:01:41.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:41 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T23:01:41.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:41 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:41.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:41 smithi022 ceph-mon[23758]: Deploying daemon osd.3 on smithi022 2024-09-13T23:01:41.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:41 smithi022 ceph-mon[28440]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 117 KiB/s wr, 8 op/s 2024-09-13T23:01:41.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:41 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T23:01:41.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:41 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:41.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:41 smithi022 ceph-mon[28440]: Deploying daemon osd.3 on smithi022 2024-09-13T23:01:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:41 smithi076 ceph-mon[24587]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 117 KiB/s wr, 8 op/s 2024-09-13T23:01:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:41 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T23:01:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:41 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:41 smithi076 ceph-mon[24587]: Deploying daemon osd.3 on smithi022 2024-09-13T23:01:43.968 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:43 smithi022 ceph-mon[23758]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.1 KiB/s rd, 90 KiB/s wr, 6 op/s 2024-09-13T23:01:43.969 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:43 smithi022 ceph-mon[28440]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.1 KiB/s rd, 90 KiB/s wr, 6 op/s 2024-09-13T23:01:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:43 smithi076 ceph-mon[24587]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.1 KiB/s rd, 90 KiB/s wr, 6 op/s 2024-09-13T23:01:45.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:45 smithi022 ceph-mon[23758]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-09-13T23:01:45.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:45 smithi022 ceph-mon[28440]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-09-13T23:01:46.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:45 smithi076 ceph-mon[24587]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-09-13T23:01:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:47 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:47 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:47 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:47 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:47.645 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:47 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:47.645 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:47 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:47.645 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:47 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:47.645 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:47 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:47.645 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:47 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:47.645 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:47 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:47.645 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:47 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:47.646 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:47 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:48.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:48 smithi022 ceph-mon[28440]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T23:01:48.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:48 smithi022 ceph-mon[23758]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T23:01:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:48 smithi076 ceph-mon[24587]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T23:01:49.548 INFO:teuthology.orchestra.run.smithi022.stdout:Created osd(s) 3 on host 'smithi022' 2024-09-13T23:01:50.137 DEBUG:teuthology.orchestra.run.smithi022:osd.3> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T23:01:50.139 INFO:tasks.cephadm:Deploying osd.4 on smithi076 with /dev/vg_nvme/lv_4... 2024-09-13T23:01:50.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T23:01:50.372 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:50 smithi022 ceph-mon[23758]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T23:01:50.373 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:50 smithi022 ceph-mon[23758]: from='osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T23:01:50.373 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:50 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:50.373 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:50 smithi022 ceph-mon[28440]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T23:01:50.373 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:50 smithi022 ceph-mon[28440]: from='osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T23:01:50.373 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:50 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:50.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:50 smithi076 ceph-mon[24587]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T23:01:50.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:50 smithi076 ceph-mon[24587]: from='osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T23:01:50.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:50 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:51.374 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:51 smithi022 ceph-mon[28440]: from='osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T23:01:51.374 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:51 smithi022 ceph-mon[28440]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T23:01:51.374 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:51.375 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:51 smithi022 ceph-mon[28440]: from='osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:01:51.375 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:51 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:51.375 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:51 smithi022 ceph-mon[23758]: from='osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T23:01:51.375 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:51 smithi022 ceph-mon[23758]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T23:01:51.375 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:51.376 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:51 smithi022 ceph-mon[23758]: from='osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:01:51.376 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:51 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:51.376 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:01:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[40115]: 2024-09-13T23:01:51.232+0000 7fa03c471700 -1 osd.3 0 waiting for initial osdmap 2024-09-13T23:01:51.376 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:01:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[40115]: 2024-09-13T23:01:51.236+0000 7fa036607700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:01:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:51 smithi076 ceph-mon[24587]: from='osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T23:01:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:51 smithi076 ceph-mon[24587]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T23:01:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:51 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:51 smithi076 ceph-mon[24587]: from='osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:01:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:51 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:52.297 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:52 smithi022 ceph-mon[23758]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:52.297 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:52 smithi022 ceph-mon[23758]: from='osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-13T23:01:52.297 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:52 smithi022 ceph-mon[23758]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T23:01:52.297 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:52 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:52.297 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:52 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:52.297 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:52 smithi022 ceph-mon[28440]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:52.297 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:52 smithi022 ceph-mon[28440]: from='osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-13T23:01:52.297 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:52 smithi022 ceph-mon[28440]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T23:01:52.297 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:52 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:52.298 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:52 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:52.481 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:52 smithi076 ceph-mon[24587]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:52.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:52 smithi076 ceph-mon[24587]: from='osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-13T23:01:52.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:52 smithi076 ceph-mon[24587]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T23:01:52.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:52 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:52.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:52 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T23:01:52.826 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi076:vg_nvme/lv_4 2024-09-13T23:01:53.447 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:53 smithi076 ceph-mon[24587]: purged_snaps scrub starts 2024-09-13T23:01:53.447 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:53 smithi076 ceph-mon[24587]: purged_snaps scrub ok 2024-09-13T23:01:53.447 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:53 smithi076 ceph-mon[24587]: osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141] boot 2024-09-13T23:01:53.447 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:53 smithi076 ceph-mon[24587]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T23:01:53.447 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:53 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:53.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:53 smithi022 ceph-mon[23758]: purged_snaps scrub starts 2024-09-13T23:01:53.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:53 smithi022 ceph-mon[23758]: purged_snaps scrub ok 2024-09-13T23:01:53.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:53 smithi022 ceph-mon[23758]: osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141] boot 2024-09-13T23:01:53.651 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:53 smithi022 ceph-mon[23758]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T23:01:53.651 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:53 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:53.651 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:53 smithi022 ceph-mon[28440]: purged_snaps scrub starts 2024-09-13T23:01:53.651 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:53 smithi022 ceph-mon[28440]: purged_snaps scrub ok 2024-09-13T23:01:53.651 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:53 smithi022 ceph-mon[28440]: osd.3 [v2:172.21.15.22:6826/2701159141,v1:172.21.15.22:6827/2701159141] boot 2024-09-13T23:01:53.651 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:53 smithi022 ceph-mon[28440]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T23:01:53.651 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:53 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:01:54.362 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:54.362 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T23:01:54.362 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:54.362 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:54.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:54.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:54.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:54.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:54.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:54.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:54.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:54.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:54.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:54.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:54.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T23:01:54.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:54.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:54.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:54.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:54.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:54.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:54.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:54.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:54.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:54.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:54.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:54.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T23:01:54.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T23:01:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:01:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:01:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:01:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:01:55.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:55 smithi076 ceph-mon[24587]: Detected new or changed devices on smithi022 2024-09-13T23:01:55.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:55 smithi076 ceph-mon[24587]: Adjusting osd_memory_target on smithi022 to 4017M 2024-09-13T23:01:55.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:55 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:01:55.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:55 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:01:55.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:55 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:55.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:55 smithi022 ceph-mon[23758]: Detected new or changed devices on smithi022 2024-09-13T23:01:55.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:55 smithi022 ceph-mon[23758]: Adjusting osd_memory_target on smithi022 to 4017M 2024-09-13T23:01:55.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:55 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:01:55.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:55 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:01:55.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:55 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:55.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:55 smithi022 ceph-mon[28440]: Detected new or changed devices on smithi022 2024-09-13T23:01:55.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:55 smithi022 ceph-mon[28440]: Adjusting osd_memory_target on smithi022 to 4017M 2024-09-13T23:01:55.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:55 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:01:55.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:55 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:01:55.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:55 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:01:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:56 smithi076 ceph-mon[24587]: from='client.34205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:01:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:56 smithi076 ceph-mon[24587]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:01:56.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:56 smithi022 ceph-mon[23758]: from='client.34205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:01:56.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:56 smithi022 ceph-mon[23758]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:01:56.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:56 smithi022 ceph-mon[28440]: from='client.34205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:01:56.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:56 smithi022 ceph-mon[28440]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:01:58.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:57 smithi076 ceph-mon[24587]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:01:58.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:57 smithi076 ceph-mon[24587]: from='client.? 172.21.15.76:0/4153311724' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f261f4b3-5255-4944-89ef-17542fcf62d3"}]: dispatch 2024-09-13T23:01:58.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:57 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f261f4b3-5255-4944-89ef-17542fcf62d3"}]: dispatch 2024-09-13T23:01:58.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:57 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f261f4b3-5255-4944-89ef-17542fcf62d3"}]': finished 2024-09-13T23:01:58.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:57 smithi076 ceph-mon[24587]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T23:01:58.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:57 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:01:58.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:57 smithi076 ceph-mon[24587]: from='client.? 172.21.15.76:0/3253173607' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:01:58.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[23758]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:01:58.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/4153311724' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f261f4b3-5255-4944-89ef-17542fcf62d3"}]: dispatch 2024-09-13T23:01:58.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f261f4b3-5255-4944-89ef-17542fcf62d3"}]: dispatch 2024-09-13T23:01:58.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f261f4b3-5255-4944-89ef-17542fcf62d3"}]': finished 2024-09-13T23:01:58.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[23758]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T23:01:58.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:01:58.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/3253173607' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:01:58.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[28440]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:01:58.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/4153311724' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f261f4b3-5255-4944-89ef-17542fcf62d3"}]: dispatch 2024-09-13T23:01:58.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f261f4b3-5255-4944-89ef-17542fcf62d3"}]: dispatch 2024-09-13T23:01:58.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f261f4b3-5255-4944-89ef-17542fcf62d3"}]': finished 2024-09-13T23:01:58.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[28440]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T23:01:58.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:01:58.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:57 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/3253173607' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:02:00.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:01:59 smithi076 ceph-mon[24587]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:00.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:01:59 smithi022 ceph-mon[28440]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:00.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:01:59 smithi022 ceph-mon[23758]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:01.848 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:01 smithi076 ceph-mon[24587]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:02.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:01 smithi022 ceph-mon[23758]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:02.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:01 smithi022 ceph-mon[28440]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:04.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:03 smithi076 ceph-mon[24587]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:04.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T23:02:04.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:03 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:04.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:03 smithi076 ceph-mon[24587]: Deploying daemon osd.4 on smithi076 2024-09-13T23:02:04.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:03 smithi022 ceph-mon[23758]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:04.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:03 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T23:02:04.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:03 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:03 smithi022 ceph-mon[23758]: Deploying daemon osd.4 on smithi076 2024-09-13T23:02:04.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:03 smithi022 ceph-mon[28440]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:04.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:03 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T23:02:04.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:03 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:04.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:03 smithi022 ceph-mon[28440]: Deploying daemon osd.4 on smithi076 2024-09-13T23:02:05.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:05 smithi076 ceph-mon[24587]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:06.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:05 smithi022 ceph-mon[23758]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:06.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:05 smithi022 ceph-mon[28440]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:08.021 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:07 smithi076 ceph-mon[24587]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:08.022 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:07 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:08.022 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:07 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:08.022 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:07 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:08.022 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:07 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:08.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:07 smithi022 ceph-mon[23758]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:08.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:07 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:08.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:07 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:08.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:07 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:08.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:07 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:07 smithi022 ceph-mon[28440]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:07 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:07 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:07 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:07 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:10.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:09 smithi076 ceph-mon[24587]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:10.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:09 smithi022 ceph-mon[23758]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:09 smithi022 ceph-mon[28440]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:11.208 INFO:teuthology.orchestra.run.smithi076.stdout:Created osd(s) 4 on host 'smithi076' 2024-09-13T23:02:11.513 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:11 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:11.513 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:11 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:11.514 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:11 smithi076 ceph-mon[24587]: from='osd.4 [v2:172.21.15.76:6800/3477942838,v1:172.21.15.76:6801/3477942838]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T23:02:11.514 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:11 smithi076 ceph-mon[24587]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T23:02:11.514 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:11 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:11.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:11 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:11.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:11 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:11 smithi022 ceph-mon[23758]: from='osd.4 [v2:172.21.15.76:6800/3477942838,v1:172.21.15.76:6801/3477942838]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T23:02:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:11 smithi022 ceph-mon[23758]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T23:02:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:11 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:11 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:11 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:11 smithi022 ceph-mon[28440]: from='osd.4 [v2:172.21.15.76:6800/3477942838,v1:172.21.15.76:6801/3477942838]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T23:02:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:11 smithi022 ceph-mon[28440]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T23:02:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:11 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:11.845 DEBUG:teuthology.orchestra.run.smithi076:osd.4> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.4.service 2024-09-13T23:02:11.848 INFO:tasks.cephadm:Deploying osd.5 on smithi076 with /dev/vg_nvme/lv_3... 2024-09-13T23:02:11.848 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T23:02:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:12 smithi076 ceph-mon[24587]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:12 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:12 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:12 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:12 smithi076 ceph-mon[24587]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T23:02:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:12 smithi076 ceph-mon[24587]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T23:02:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:12 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:02:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:12 smithi076 ceph-mon[24587]: from='osd.4 [v2:172.21.15.76:6800/3477942838,v1:172.21.15.76:6801/3477942838]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:02:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:12 smithi076 ceph-mon[24587]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:02:12.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[23758]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[23758]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T23:02:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[23758]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T23:02:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:02:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[23758]: from='osd.4 [v2:172.21.15.76:6800/3477942838,v1:172.21.15.76:6801/3477942838]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:02:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[23758]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:02:12.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[28440]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:12.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:12.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:12.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:12.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[28440]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T23:02:12.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[28440]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T23:02:12.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:02:12.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[28440]: from='osd.4 [v2:172.21.15.76:6800/3477942838,v1:172.21.15.76:6801/3477942838]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:02:12.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:12 smithi022 ceph-mon[28440]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:02:12.787 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:02:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[27943]: 2024-09-13T23:02:12.534+0000 7f1b19e7b700 -1 osd.4 0 waiting for initial osdmap 2024-09-13T23:02:12.787 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:02:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[27943]: 2024-09-13T23:02:12.538+0000 7f1b14011700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:02:13.751 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:13 smithi076 ceph-mon[24587]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-13T23:02:13.751 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:13 smithi076 ceph-mon[24587]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T23:02:13.751 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:13 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:02:13.751 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:13 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:02:13.751 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:13 smithi076 ceph-mon[24587]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:13.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:13 smithi022 ceph-mon[23758]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-13T23:02:13.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:13 smithi022 ceph-mon[23758]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T23:02:13.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:13 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:02:13.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:13 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:02:13.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:13 smithi022 ceph-mon[23758]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:13.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:13 smithi022 ceph-mon[28440]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-13T23:02:13.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:13 smithi022 ceph-mon[28440]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T23:02:13.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:13 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:02:13.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:13 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:02:13.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:13 smithi022 ceph-mon[28440]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T23:02:14.723 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:14 smithi076 ceph-mon[24587]: purged_snaps scrub starts 2024-09-13T23:02:14.723 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:14 smithi076 ceph-mon[24587]: purged_snaps scrub ok 2024-09-13T23:02:14.723 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:14 smithi076 ceph-mon[24587]: osd.4 [v2:172.21.15.76:6800/3477942838,v1:172.21.15.76:6801/3477942838] boot 2024-09-13T23:02:14.723 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:14 smithi076 ceph-mon[24587]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T23:02:14.723 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:14 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:02:14.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:14 smithi022 ceph-mon[23758]: purged_snaps scrub starts 2024-09-13T23:02:14.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:14 smithi022 ceph-mon[23758]: purged_snaps scrub ok 2024-09-13T23:02:14.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:14 smithi022 ceph-mon[23758]: osd.4 [v2:172.21.15.76:6800/3477942838,v1:172.21.15.76:6801/3477942838] boot 2024-09-13T23:02:14.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:14 smithi022 ceph-mon[23758]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T23:02:14.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:14 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:02:14.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:14 smithi022 ceph-mon[28440]: purged_snaps scrub starts 2024-09-13T23:02:14.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:14 smithi022 ceph-mon[28440]: purged_snaps scrub ok 2024-09-13T23:02:14.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:14 smithi022 ceph-mon[28440]: osd.4 [v2:172.21.15.76:6800/3477942838,v1:172.21.15.76:6801/3477942838] boot 2024-09-13T23:02:14.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:14 smithi022 ceph-mon[28440]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T23:02:14.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:14 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:02:15.045 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T23:02:15.073 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi076:vg_nvme/lv_3 2024-09-13T23:02:15.576 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:15 smithi076 ceph-mon[24587]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T23:02:15.576 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:15 smithi076 ceph-mon[24587]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:15.576 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:15 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:02:15.577 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:15 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:02:15.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:15 smithi022 ceph-mon[23758]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T23:02:15.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:15 smithi022 ceph-mon[23758]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:15.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:02:15.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:15 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:02:15.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:15 smithi022 ceph-mon[28440]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T23:02:15.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:15 smithi022 ceph-mon[28440]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:15.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:15 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:02:15.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:15 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:02:16.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:16 smithi076 ceph-mon[24587]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T23:02:16.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:16 smithi022 ceph-mon[23758]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T23:02:16.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:16 smithi022 ceph-mon[28440]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T23:02:17.864 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:17 smithi076 ceph-mon[24587]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T23:02:17.864 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:17 smithi076 ceph-mon[24587]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:17.864 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:17 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:02:17.864 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:17 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:02:17.864 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:17 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:17.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:17 smithi022 ceph-mon[23758]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T23:02:17.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:17 smithi022 ceph-mon[23758]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:17.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:17 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:02:17.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:17 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:02:17.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:17 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:17 smithi022 ceph-mon[28440]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T23:02:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:17 smithi022 ceph-mon[28440]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:17 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:02:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:17 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:02:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:17 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:18 smithi076 ceph-mon[24587]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:02:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:18 smithi076 ceph-mon[24587]: osdmap e35: 5 total, 5 up, 5 in 2024-09-13T23:02:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:18 smithi076 ceph-mon[24587]: Detected new or changed devices on smithi076 2024-09-13T23:02:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:18 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:18 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:02:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:18 smithi076 ceph-mon[24587]: Adjusting osd_memory_target on smithi076 to 17088M 2024-09-13T23:02:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:18 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:18 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:18.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[23758]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:02:18.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[23758]: osdmap e35: 5 total, 5 up, 5 in 2024-09-13T23:02:18.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[23758]: Detected new or changed devices on smithi076 2024-09-13T23:02:18.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:18.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:02:18.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[23758]: Adjusting osd_memory_target on smithi076 to 17088M 2024-09-13T23:02:18.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:18.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[28440]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:02:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[28440]: osdmap e35: 5 total, 5 up, 5 in 2024-09-13T23:02:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[28440]: Detected new or changed devices on smithi076 2024-09-13T23:02:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:02:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[28440]: Adjusting osd_memory_target on smithi076 to 17088M 2024-09-13T23:02:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:18.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:18 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:19.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:19 smithi022 ceph-mon[23758]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 125 KiB/s, 0 objects/s recovering 2024-09-13T23:02:19.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:19 smithi022 ceph-mon[28440]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 125 KiB/s, 0 objects/s recovering 2024-09-13T23:02:20.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:19 smithi076 ceph-mon[24587]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 125 KiB/s, 0 objects/s recovering 2024-09-13T23:02:20.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:20 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/1815255921' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b53bd7f5-fc30-4e46-8deb-43ab8f0c10c2"}]: dispatch 2024-09-13T23:02:20.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:20 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b53bd7f5-fc30-4e46-8deb-43ab8f0c10c2"}]: dispatch 2024-09-13T23:02:20.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:20 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b53bd7f5-fc30-4e46-8deb-43ab8f0c10c2"}]': finished 2024-09-13T23:02:20.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:20 smithi022 ceph-mon[23758]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T23:02:20.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:20 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:20.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:20 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/337559560' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:02:20.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:20 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/1815255921' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b53bd7f5-fc30-4e46-8deb-43ab8f0c10c2"}]: dispatch 2024-09-13T23:02:20.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:20 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b53bd7f5-fc30-4e46-8deb-43ab8f0c10c2"}]: dispatch 2024-09-13T23:02:20.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:20 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b53bd7f5-fc30-4e46-8deb-43ab8f0c10c2"}]': finished 2024-09-13T23:02:20.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:20 smithi022 ceph-mon[28440]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T23:02:20.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:20 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:20.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:20 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/337559560' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:02:21.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:20 smithi076 ceph-mon[24587]: from='client.? 172.21.15.76:0/1815255921' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b53bd7f5-fc30-4e46-8deb-43ab8f0c10c2"}]: dispatch 2024-09-13T23:02:21.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:20 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b53bd7f5-fc30-4e46-8deb-43ab8f0c10c2"}]: dispatch 2024-09-13T23:02:21.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:20 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b53bd7f5-fc30-4e46-8deb-43ab8f0c10c2"}]': finished 2024-09-13T23:02:21.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:20 smithi076 ceph-mon[24587]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T23:02:21.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:20 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:21.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:20 smithi076 ceph-mon[24587]: from='client.? 172.21.15.76:0/337559560' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:02:21.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:21 smithi022 ceph-mon[23758]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-09-13T23:02:21.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:21 smithi022 ceph-mon[28440]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-09-13T23:02:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:21 smithi076 ceph-mon[24587]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-09-13T23:02:23.979 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:23 smithi076 ceph-mon[24587]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 85 KiB/s, 0 objects/s recovering 2024-09-13T23:02:24.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:23 smithi022 ceph-mon[23758]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 85 KiB/s, 0 objects/s recovering 2024-09-13T23:02:24.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:23 smithi022 ceph-mon[28440]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 85 KiB/s, 0 objects/s recovering 2024-09-13T23:02:25.930 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:25 smithi076 ceph-mon[24587]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T23:02:26.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:25 smithi022 ceph-mon[23758]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T23:02:26.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:25 smithi022 ceph-mon[28440]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T23:02:27.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:26 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T23:02:27.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:26 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:27.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:26 smithi076 ceph-mon[24587]: Deploying daemon osd.5 on smithi076 2024-09-13T23:02:27.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:26 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T23:02:27.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:26 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:27.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:26 smithi022 ceph-mon[23758]: Deploying daemon osd.5 on smithi076 2024-09-13T23:02:27.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:26 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T23:02:27.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:26 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:27.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:26 smithi022 ceph-mon[28440]: Deploying daemon osd.5 on smithi076 2024-09-13T23:02:27.890 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:27 smithi076 ceph-mon[24587]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-09-13T23:02:28.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:27 smithi022 ceph-mon[23758]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-09-13T23:02:28.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:27 smithi022 ceph-mon[28440]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-09-13T23:02:30.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:29 smithi076 ceph-mon[24587]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:30.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:29 smithi022 ceph-mon[23758]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:30.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:29 smithi022 ceph-mon[28440]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:32.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:31 smithi076 ceph-mon[24587]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:32.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:32.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:32.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:32.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:31 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:32.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:31 smithi022 ceph-mon[23758]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:32.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:32.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:32.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:32.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:31 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:32.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:31 smithi022 ceph-mon[28440]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:32.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:32.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:32.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:32.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:31 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:34.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:33 smithi076 ceph-mon[24587]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:34.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:33 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:34.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:33 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:34.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:33 smithi022 ceph-mon[23758]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:34.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:33 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:34.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:33 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:33 smithi022 ceph-mon[28440]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:33 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:33 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:34.372 INFO:teuthology.orchestra.run.smithi076.stdout:Created osd(s) 5 on host 'smithi076' 2024-09-13T23:02:35.003 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:34 smithi076 ceph-mon[24587]: from='osd.5 [v2:172.21.15.76:6808/2219212499,v1:172.21.15.76:6809/2219212499]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T23:02:35.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:34 smithi076 ceph-mon[24587]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T23:02:35.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:34 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:35.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:34 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:35.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:34 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:35.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:34 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:35.026 DEBUG:teuthology.orchestra.run.smithi076:osd.5> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T23:02:35.030 INFO:tasks.cephadm:Deploying osd.6 on smithi076 with /dev/vg_nvme/lv_2... 2024-09-13T23:02:35.030 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T23:02:35.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:34 smithi022 ceph-mon[23758]: from='osd.5 [v2:172.21.15.76:6808/2219212499,v1:172.21.15.76:6809/2219212499]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T23:02:35.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:34 smithi022 ceph-mon[23758]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T23:02:35.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:34 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:35.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:34 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:35.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:34 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:35.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:34 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:35.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:34 smithi022 ceph-mon[28440]: from='osd.5 [v2:172.21.15.76:6808/2219212499,v1:172.21.15.76:6809/2219212499]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T23:02:35.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:34 smithi022 ceph-mon[28440]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T23:02:35.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:34 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:35.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:34 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:35.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:34 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:35.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:34 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:36.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:35 smithi076 ceph-mon[24587]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:36.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:35 smithi076 ceph-mon[24587]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T23:02:36.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:35 smithi076 ceph-mon[24587]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T23:02:36.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:35 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:36.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:35 smithi076 ceph-mon[24587]: from='osd.5 [v2:172.21.15.76:6808/2219212499,v1:172.21.15.76:6809/2219212499]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:02:36.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:35 smithi076 ceph-mon[24587]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:02:36.038 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:02:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[30866]: 2024-09-13T23:02:35.896+0000 7f862ec2f700 -1 osd.5 0 waiting for initial osdmap 2024-09-13T23:02:36.038 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:02:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[30866]: 2024-09-13T23:02:35.901+0000 7f862b5ca700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:02:36.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:35 smithi022 ceph-mon[23758]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:36.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:35 smithi022 ceph-mon[23758]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T23:02:36.324 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:35 smithi022 ceph-mon[23758]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T23:02:36.324 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:35 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:36.324 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:35 smithi022 ceph-mon[23758]: from='osd.5 [v2:172.21.15.76:6808/2219212499,v1:172.21.15.76:6809/2219212499]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:02:36.324 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:35 smithi022 ceph-mon[23758]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:02:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:35 smithi022 ceph-mon[28440]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:35 smithi022 ceph-mon[28440]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T23:02:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:35 smithi022 ceph-mon[28440]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T23:02:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:35 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:35 smithi022 ceph-mon[28440]: from='osd.5 [v2:172.21.15.76:6808/2219212499,v1:172.21.15.76:6809/2219212499]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:02:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:35 smithi022 ceph-mon[28440]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:02:36.957 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:36 smithi076 ceph-mon[24587]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-13T23:02:36.957 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:36 smithi076 ceph-mon[24587]: osdmap e38: 6 total, 5 up, 6 in 2024-09-13T23:02:36.957 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:36 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:36.957 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:36 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:36.957 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:36 smithi076 ceph-mon[24587]: osd.5 [v2:172.21.15.76:6808/2219212499,v1:172.21.15.76:6809/2219212499] boot 2024-09-13T23:02:36.957 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:36 smithi076 ceph-mon[24587]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T23:02:36.957 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:36 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:37.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[23758]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-13T23:02:37.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[23758]: osdmap e38: 6 total, 5 up, 6 in 2024-09-13T23:02:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[23758]: osd.5 [v2:172.21.15.76:6808/2219212499,v1:172.21.15.76:6809/2219212499] boot 2024-09-13T23:02:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[23758]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T23:02:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:37.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[28440]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-13T23:02:37.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[28440]: osdmap e38: 6 total, 5 up, 6 in 2024-09-13T23:02:37.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:37.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:37.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[28440]: osd.5 [v2:172.21.15.76:6808/2219212499,v1:172.21.15.76:6809/2219212499] boot 2024-09-13T23:02:37.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[28440]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T23:02:37.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:36 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:02:37.994 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:37 smithi076 ceph-mon[24587]: purged_snaps scrub starts 2024-09-13T23:02:37.994 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:37 smithi076 ceph-mon[24587]: purged_snaps scrub ok 2024-09-13T23:02:37.994 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:37 smithi076 ceph-mon[24587]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:37.994 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:37 smithi076 ceph-mon[24587]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T23:02:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:37 smithi022 ceph-mon[23758]: purged_snaps scrub starts 2024-09-13T23:02:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:37 smithi022 ceph-mon[23758]: purged_snaps scrub ok 2024-09-13T23:02:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:37 smithi022 ceph-mon[23758]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:37 smithi022 ceph-mon[23758]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T23:02:38.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:37 smithi022 ceph-mon[28440]: purged_snaps scrub starts 2024-09-13T23:02:38.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:37 smithi022 ceph-mon[28440]: purged_snaps scrub ok 2024-09-13T23:02:38.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:37 smithi022 ceph-mon[28440]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T23:02:38.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:37 smithi022 ceph-mon[28440]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T23:02:39.763 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:39 smithi076 ceph-mon[24587]: Detected new or changed devices on smithi076 2024-09-13T23:02:39.763 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:39.763 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:02:39.763 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:02:39.763 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:39 smithi076 ceph-mon[24587]: Adjusting osd_memory_target on smithi076 to 8544M 2024-09-13T23:02:39.763 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:39.764 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:39.764 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:39 smithi076 ceph-mon[24587]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:39.764 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:39 smithi076 ceph-mon[24587]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T23:02:39.767 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T23:02:39.786 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi076:vg_nvme/lv_2 2024-09-13T23:02:39.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[28440]: Detected new or changed devices on smithi076 2024-09-13T23:02:39.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:39.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:02:39.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:02:39.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[28440]: Adjusting osd_memory_target on smithi076 to 8544M 2024-09-13T23:02:39.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:39.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:39.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[28440]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:39.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[28440]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T23:02:39.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[23758]: Detected new or changed devices on smithi076 2024-09-13T23:02:39.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:39.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:02:39.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:02:39.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[23758]: Adjusting osd_memory_target on smithi076 to 8544M 2024-09-13T23:02:39.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:39.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:39.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[23758]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:39.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:39 smithi022 ceph-mon[23758]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T23:02:41.026 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:40 smithi076 ceph-mon[24587]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T23:02:41.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:40 smithi022 ceph-mon[23758]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T23:02:41.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:40 smithi022 ceph-mon[28440]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T23:02:42.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:41 smithi076 ceph-mon[24587]: pgmap v135: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:42.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:41 smithi076 ceph-mon[24587]: osdmap e43: 6 total, 6 up, 6 in 2024-09-13T23:02:42.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:41 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:02:42.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:41 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:02:42.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:41 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:42.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:41 smithi022 ceph-mon[23758]: pgmap v135: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:42.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:41 smithi022 ceph-mon[23758]: osdmap e43: 6 total, 6 up, 6 in 2024-09-13T23:02:42.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:41 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:02:42.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:41 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:02:42.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:41 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:42.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:41 smithi022 ceph-mon[28440]: pgmap v135: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:42.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:41 smithi022 ceph-mon[28440]: osdmap e43: 6 total, 6 up, 6 in 2024-09-13T23:02:42.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:41 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:02:42.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:41 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:02:42.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:41 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:42.985 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:42 smithi076 ceph-mon[24587]: from='client.34259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:02:43.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:42 smithi022 ceph-mon[23758]: from='client.34259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:02:43.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:42 smithi022 ceph-mon[28440]: from='client.34259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:02:44.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:43 smithi076 ceph-mon[24587]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:44.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:43 smithi022 ceph-mon[23758]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:44.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:43 smithi022 ceph-mon[28440]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:45.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:44 smithi076 ceph-mon[24587]: from='client.? 172.21.15.76:0/1062175296' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8901b439-aa14-481e-a3f1-341fe4c467e9"}]: dispatch 2024-09-13T23:02:45.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:44 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8901b439-aa14-481e-a3f1-341fe4c467e9"}]: dispatch 2024-09-13T23:02:45.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:44 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8901b439-aa14-481e-a3f1-341fe4c467e9"}]': finished 2024-09-13T23:02:45.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:44 smithi076 ceph-mon[24587]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T23:02:45.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:44 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:02:45.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:44 smithi076 ceph-mon[24587]: from='client.? 172.21.15.76:0/755368413' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:02:45.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:44 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/1062175296' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8901b439-aa14-481e-a3f1-341fe4c467e9"}]: dispatch 2024-09-13T23:02:45.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:44 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8901b439-aa14-481e-a3f1-341fe4c467e9"}]: dispatch 2024-09-13T23:02:45.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:44 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8901b439-aa14-481e-a3f1-341fe4c467e9"}]': finished 2024-09-13T23:02:45.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:44 smithi022 ceph-mon[23758]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T23:02:45.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:44 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:02:45.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:44 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/755368413' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:02:45.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:44 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/1062175296' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8901b439-aa14-481e-a3f1-341fe4c467e9"}]: dispatch 2024-09-13T23:02:45.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:44 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8901b439-aa14-481e-a3f1-341fe4c467e9"}]: dispatch 2024-09-13T23:02:45.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:44 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8901b439-aa14-481e-a3f1-341fe4c467e9"}]': finished 2024-09-13T23:02:45.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:44 smithi022 ceph-mon[28440]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T23:02:45.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:44 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:02:45.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:44 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/755368413' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:02:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:45 smithi076 ceph-mon[24587]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:46.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:45 smithi022 ceph-mon[23758]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:46.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:45 smithi022 ceph-mon[28440]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:48.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:47 smithi076 ceph-mon[24587]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:47 smithi022 ceph-mon[23758]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:48.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:47 smithi022 ceph-mon[28440]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:49 smithi076 ceph-mon[24587]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:50.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:49 smithi022 ceph-mon[23758]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:50.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:49 smithi022 ceph-mon[28440]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:51.185 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:50 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T23:02:51.185 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:50 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:51.185 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:50 smithi076 ceph-mon[24587]: Deploying daemon osd.6 on smithi076 2024-09-13T23:02:51.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:50 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T23:02:51.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:50 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:50 smithi022 ceph-mon[23758]: Deploying daemon osd.6 on smithi076 2024-09-13T23:02:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:50 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T23:02:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:50 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:50 smithi022 ceph-mon[28440]: Deploying daemon osd.6 on smithi076 2024-09-13T23:02:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:51 smithi076 ceph-mon[24587]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:52.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:51 smithi022 ceph-mon[23758]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:52.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:51 smithi022 ceph-mon[28440]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:54.196 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:53 smithi076 ceph-mon[24587]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:54.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:53 smithi022 ceph-mon[23758]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:54.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:53 smithi022 ceph-mon[28440]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:55.264 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:55.264 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:55.264 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:55.264 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:54 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:55.304 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:55.304 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:55.304 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:55.304 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:54 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:55.304 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:02:55.304 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:55.304 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:02:55.305 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:54 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:02:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:55 smithi076 ceph-mon[24587]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:56.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:55 smithi022 ceph-mon[23758]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:56.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:55 smithi022 ceph-mon[28440]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:58.107 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:58 smithi076 ceph-mon[24587]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:58.146 INFO:teuthology.orchestra.run.smithi076.stdout:Created osd(s) 6 on host 'smithi076' 2024-09-13T23:02:58.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:57 smithi022 ceph-mon[23758]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:58.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:58 smithi022 ceph-mon[28440]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:02:58.805 DEBUG:teuthology.orchestra.run.smithi076:osd.6> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.6.service 2024-09-13T23:02:58.808 INFO:tasks.cephadm:Deploying osd.7 on smithi076 with /dev/vg_nvme/lv_1... 2024-09-13T23:02:58.808 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T23:02:59.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:59 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:59.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:59 smithi022 ceph-mon[23758]: from='osd.6 [v2:172.21.15.76:6816/1379127534,v1:172.21.15.76:6817/1379127534]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T23:02:59.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:59 smithi022 ceph-mon[23758]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T23:02:59.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:02:59 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:59.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:59 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:59.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:59 smithi022 ceph-mon[28440]: from='osd.6 [v2:172.21.15.76:6816/1379127534,v1:172.21.15.76:6817/1379127534]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T23:02:59.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:59 smithi022 ceph-mon[28440]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T23:02:59.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:02:59 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:59.464 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:59 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:02:59.464 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:59 smithi076 ceph-mon[24587]: from='osd.6 [v2:172.21.15.76:6816/1379127534,v1:172.21.15.76:6817/1379127534]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T23:02:59.464 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:59 smithi076 ceph-mon[24587]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T23:02:59.464 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:02:59 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:00.192 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:00 smithi076 ceph-mon[24587]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:03:00.192 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:00 smithi076 ceph-mon[24587]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T23:03:00.192 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:00 smithi076 ceph-mon[24587]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T23:03:00.192 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:00 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:00.193 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:00 smithi076 ceph-mon[24587]: from='osd.6 [v2:172.21.15.76:6816/1379127534,v1:172.21.15.76:6817/1379127534]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:03:00.193 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:00 smithi076 ceph-mon[24587]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:03:00.193 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:03:00 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[33726]: 2024-09-13T23:03:00.155+0000 7fb1acc1b700 -1 osd.6 0 waiting for initial osdmap 2024-09-13T23:03:00.193 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:03:00 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[33726]: 2024-09-13T23:03:00.162+0000 7fb1a5daf700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:03:00.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:00 smithi022 ceph-mon[23758]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:03:00.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:00 smithi022 ceph-mon[23758]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T23:03:00.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:00 smithi022 ceph-mon[23758]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T23:03:00.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:00 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:00.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:00 smithi022 ceph-mon[23758]: from='osd.6 [v2:172.21.15.76:6816/1379127534,v1:172.21.15.76:6817/1379127534]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:03:00.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:00 smithi022 ceph-mon[23758]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:03:00.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:00 smithi022 ceph-mon[28440]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:03:00.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:00 smithi022 ceph-mon[28440]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T23:03:00.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:00 smithi022 ceph-mon[28440]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T23:03:00.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:00 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:00.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:00 smithi022 ceph-mon[28440]: from='osd.6 [v2:172.21.15.76:6816/1379127534,v1:172.21.15.76:6817/1379127534]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:03:00.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:00 smithi022 ceph-mon[28440]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:03:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:01 smithi022 ceph-mon[23758]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-13T23:03:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:01 smithi022 ceph-mon[23758]: osdmap e46: 7 total, 6 up, 7 in 2024-09-13T23:03:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:01 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:01 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:01 smithi022 ceph-mon[28440]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-13T23:03:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:01 smithi022 ceph-mon[28440]: osdmap e46: 7 total, 6 up, 7 in 2024-09-13T23:03:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:01 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:01 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:01 smithi076 ceph-mon[24587]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-13T23:03:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:01 smithi076 ceph-mon[24587]: osdmap e46: 7 total, 6 up, 7 in 2024-09-13T23:03:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:01 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:01 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:02 smithi076 ceph-mon[24587]: purged_snaps scrub starts 2024-09-13T23:03:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:02 smithi076 ceph-mon[24587]: purged_snaps scrub ok 2024-09-13T23:03:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:02 smithi076 ceph-mon[24587]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:03:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:02 smithi076 ceph-mon[24587]: osd.6 [v2:172.21.15.76:6816/1379127534,v1:172.21.15.76:6817/1379127534] boot 2024-09-13T23:03:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:02 smithi076 ceph-mon[24587]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T23:03:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:02 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:02 smithi076 ceph-mon[24587]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T23:03:02.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[23758]: purged_snaps scrub starts 2024-09-13T23:03:02.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[23758]: purged_snaps scrub ok 2024-09-13T23:03:02.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[23758]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:03:02.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[23758]: osd.6 [v2:172.21.15.76:6816/1379127534,v1:172.21.15.76:6817/1379127534] boot 2024-09-13T23:03:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[23758]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T23:03:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[23758]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T23:03:02.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[28440]: purged_snaps scrub starts 2024-09-13T23:03:02.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[28440]: purged_snaps scrub ok 2024-09-13T23:03:02.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[28440]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T23:03:02.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[28440]: osd.6 [v2:172.21.15.76:6816/1379127534,v1:172.21.15.76:6817/1379127534] boot 2024-09-13T23:03:02.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[28440]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T23:03:02.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:02.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:02 smithi022 ceph-mon[28440]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T23:03:02.626 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-13T23:03:02.645 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi076:vg_nvme/lv_1 2024-09-13T23:03:04.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:03 smithi076 ceph-mon[24587]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:04.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:03 smithi076 ceph-mon[24587]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T23:03:04.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:03 smithi022 ceph-mon[23758]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:04.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:03 smithi022 ceph-mon[23758]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T23:03:04.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:03 smithi022 ceph-mon[28440]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:04.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:03 smithi022 ceph-mon[28440]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T23:03:04.987 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:04 smithi076 ceph-mon[24587]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T23:03:05.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:04 smithi022 ceph-mon[23758]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T23:03:05.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:04 smithi022 ceph-mon[28440]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T23:03:06.003 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:06.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: osdmap e51: 7 total, 7 up, 7 in 2024-09-13T23:03:06.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: Detected new or changed devices on smithi076 2024-09-13T23:03:06.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:06.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:06.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:06.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:06.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: Adjusting osd_memory_target on smithi076 to 5696M 2024-09-13T23:03:06.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:06.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:06.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:06.004 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:06.005 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:06.005 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:06.005 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:03:06.005 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:03:06.005 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:05 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:06.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:06.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: osdmap e51: 7 total, 7 up, 7 in 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: Detected new or changed devices on smithi076 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: Adjusting osd_memory_target on smithi076 to 5696M 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:03:06.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:03:06.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:06.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:06.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: osdmap e51: 7 total, 7 up, 7 in 2024-09-13T23:03:06.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: Detected new or changed devices on smithi076 2024-09-13T23:03:06.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:06.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:06.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:06.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:06.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: Adjusting osd_memory_target on smithi076 to 5696M 2024-09-13T23:03:06.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:06.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:06.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:06.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:06.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:06.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:06.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T23:03:06.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T23:03:06.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:05 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:07.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:06 smithi076 ceph-mon[24587]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:03:07.081 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:06 smithi022 ceph-mon[23758]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:03:07.081 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:06 smithi022 ceph-mon[28440]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:03:08.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:07 smithi076 ceph-mon[24587]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-13T23:03:08.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:07 smithi022 ceph-mon[23758]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-13T23:03:08.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:07 smithi022 ceph-mon[28440]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-13T23:03:09.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:08 smithi076 ceph-mon[24587]: from='client.? 172.21.15.76:0/772664871' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad6c9aa3-0472-4806-b96c-161a1cb9005b"}]: dispatch 2024-09-13T23:03:09.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:08 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad6c9aa3-0472-4806-b96c-161a1cb9005b"}]: dispatch 2024-09-13T23:03:09.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:08 smithi076 ceph-mon[24587]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad6c9aa3-0472-4806-b96c-161a1cb9005b"}]': finished 2024-09-13T23:03:09.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:08 smithi076 ceph-mon[24587]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T23:03:09.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:08 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:09.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:08 smithi076 ceph-mon[24587]: from='client.? 172.21.15.76:0/1830402019' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:03:09.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:08 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/772664871' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad6c9aa3-0472-4806-b96c-161a1cb9005b"}]: dispatch 2024-09-13T23:03:09.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:08 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad6c9aa3-0472-4806-b96c-161a1cb9005b"}]: dispatch 2024-09-13T23:03:09.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:08 smithi022 ceph-mon[23758]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad6c9aa3-0472-4806-b96c-161a1cb9005b"}]': finished 2024-09-13T23:03:09.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:08 smithi022 ceph-mon[23758]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T23:03:09.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:08 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:09.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:08 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/1830402019' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:03:09.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:08 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/772664871' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad6c9aa3-0472-4806-b96c-161a1cb9005b"}]: dispatch 2024-09-13T23:03:09.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:08 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad6c9aa3-0472-4806-b96c-161a1cb9005b"}]: dispatch 2024-09-13T23:03:09.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:08 smithi022 ceph-mon[28440]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad6c9aa3-0472-4806-b96c-161a1cb9005b"}]': finished 2024-09-13T23:03:09.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:08 smithi022 ceph-mon[28440]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T23:03:09.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:08 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:09.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:08 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/1830402019' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T23:03:10.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:09 smithi022 ceph-mon[23758]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-13T23:03:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:09 smithi022 ceph-mon[28440]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-13T23:03:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:09 smithi076 ceph-mon[24587]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-13T23:03:11.982 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:11 smithi076 ceph-mon[24587]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-13T23:03:12.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:11 smithi022 ceph-mon[23758]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-13T23:03:12.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:11 smithi022 ceph-mon[28440]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-13T23:03:14.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:13 smithi022 ceph-mon[23758]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T23:03:14.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:13 smithi022 ceph-mon[28440]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T23:03:14.218 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:13 smithi076 ceph-mon[24587]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T23:03:14.930 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:14 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T23:03:14.930 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:14 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:14.930 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:14 smithi076 ceph-mon[24587]: Deploying daemon osd.7 on smithi076 2024-09-13T23:03:14.931 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:14 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:03:14.931 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:14 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:03:15.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:14 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T23:03:15.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:14 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:15.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:14 smithi022 ceph-mon[23758]: Deploying daemon osd.7 on smithi076 2024-09-13T23:03:15.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:14 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:03:15.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:14 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:03:15.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:14 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T23:03:15.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:14 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:15.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:14 smithi022 ceph-mon[28440]: Deploying daemon osd.7 on smithi076 2024-09-13T23:03:15.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:14 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:03:15.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:14 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:03:16.100 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:15 smithi076 ceph-mon[24587]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T23:03:16.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:15 smithi022 ceph-mon[23758]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T23:03:16.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:15 smithi022 ceph-mon[28440]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T23:03:17.999 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:17 smithi076 ceph-mon[24587]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:18.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:17 smithi022 ceph-mon[23758]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:18.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:17 smithi022 ceph-mon[28440]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:20.480 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:20 smithi076 ceph-mon[24587]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:20.480 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:20 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:20.480 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:20 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:20.480 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:20 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:20.480 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:20 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:20.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:20 smithi022 ceph-mon[23758]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:20.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:20 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:20.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:20 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:20 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:20 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:20.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:20 smithi022 ceph-mon[28440]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:20.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:20 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:20.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:20 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:20.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:20 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:20.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:20 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:22.225 INFO:teuthology.orchestra.run.smithi076.stdout:Created osd(s) 7 on host 'smithi076' 2024-09-13T23:03:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:22 smithi076 ceph-mon[24587]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:22.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:22 smithi022 ceph-mon[23758]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:22.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:22 smithi022 ceph-mon[28440]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:22.913 DEBUG:teuthology.orchestra.run.smithi076:osd.7> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.7.service 2024-09-13T23:03:22.915 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-13T23:03:22.916 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T23:03:23.227 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:23 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:23.227 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:23 smithi022 ceph-mon[28440]: from='osd.7 [v2:172.21.15.76:6824/2500464355,v1:172.21.15.76:6825/2500464355]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T23:03:23.227 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:23 smithi022 ceph-mon[28440]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T23:03:23.227 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:23 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:23.488 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:23 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:23.489 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:23 smithi022 ceph-mon[23758]: from='osd.7 [v2:172.21.15.76:6824/2500464355,v1:172.21.15.76:6825/2500464355]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T23:03:23.489 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:23 smithi022 ceph-mon[23758]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T23:03:23.489 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:23 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:23 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:23 smithi076 ceph-mon[24587]: from='osd.7 [v2:172.21.15.76:6824/2500464355,v1:172.21.15.76:6825/2500464355]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T23:03:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:23 smithi076 ceph-mon[24587]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T23:03:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:23 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:24 smithi076 ceph-mon[24587]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:24 smithi076 ceph-mon[24587]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T23:03:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:24 smithi076 ceph-mon[24587]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T23:03:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:24 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:24 smithi076 ceph-mon[24587]: from='osd.7 [v2:172.21.15.76:6824/2500464355,v1:172.21.15.76:6825/2500464355]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:03:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:24 smithi076 ceph-mon[24587]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:03:24.538 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:03:24 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[36633]: 2024-09-13T23:03:24.235+0000 7f1def743700 -1 osd.7 0 waiting for initial osdmap 2024-09-13T23:03:24.538 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:03:24 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[36633]: 2024-09-13T23:03:24.243+0000 7f1dea8db700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:03:24.593 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:24 smithi022 ceph-mon[23758]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:24.594 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:24 smithi022 ceph-mon[23758]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T23:03:24.594 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:24 smithi022 ceph-mon[23758]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T23:03:24.594 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:24 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:24.594 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:24 smithi022 ceph-mon[23758]: from='osd.7 [v2:172.21.15.76:6824/2500464355,v1:172.21.15.76:6825/2500464355]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:03:24.594 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:24 smithi022 ceph-mon[23758]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:03:24.594 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:24 smithi022 ceph-mon[28440]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:24.594 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:24 smithi022 ceph-mon[28440]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T23:03:24.594 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:24 smithi022 ceph-mon[28440]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T23:03:24.594 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:24 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:24.595 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:24 smithi022 ceph-mon[28440]: from='osd.7 [v2:172.21.15.76:6824/2500464355,v1:172.21.15.76:6825/2500464355]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:03:24.595 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:24 smithi022 ceph-mon[28440]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:03:25.155 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:03:25.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[28440]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-13T23:03:25.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[28440]: osdmap e54: 8 total, 7 up, 8 in 2024-09-13T23:03:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/4184523991' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T23:03:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[28440]: osd.7 [v2:172.21.15.76:6824/2500464355,v1:172.21.15.76:6825/2500464355] boot 2024-09-13T23:03:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[28440]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T23:03:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:25.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[23758]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-13T23:03:25.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[23758]: osdmap e54: 8 total, 7 up, 8 in 2024-09-13T23:03:25.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:25.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:25.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/4184523991' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T23:03:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[23758]: osd.7 [v2:172.21.15.76:6824/2500464355,v1:172.21.15.76:6825/2500464355] boot 2024-09-13T23:03:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[23758]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T23:03:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:25 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:25 smithi076 ceph-mon[24587]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-13T23:03:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:25 smithi076 ceph-mon[24587]: osdmap e54: 8 total, 7 up, 8 in 2024-09-13T23:03:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:25 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:25 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:25 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/4184523991' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T23:03:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:25 smithi076 ceph-mon[24587]: osd.7 [v2:172.21.15.76:6824/2500464355,v1:172.21.15.76:6825/2500464355] boot 2024-09-13T23:03:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:25 smithi076 ceph-mon[24587]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T23:03:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:25 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:26.038 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1726268581,"num_in_osds":8,"osd_in_since":1726268587,"num_remapped_pgs":0} 2024-09-13T23:03:26.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:26 smithi022 ceph-mon[23758]: purged_snaps scrub starts 2024-09-13T23:03:26.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:26 smithi022 ceph-mon[23758]: purged_snaps scrub ok 2024-09-13T23:03:26.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:26 smithi022 ceph-mon[23758]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:26.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:26 smithi022 ceph-mon[23758]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T23:03:26.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:26 smithi022 ceph-mon[28440]: purged_snaps scrub starts 2024-09-13T23:03:26.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:26 smithi022 ceph-mon[28440]: purged_snaps scrub ok 2024-09-13T23:03:26.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:26 smithi022 ceph-mon[28440]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:26.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:26 smithi022 ceph-mon[28440]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T23:03:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:26 smithi076 ceph-mon[24587]: purged_snaps scrub starts 2024-09-13T23:03:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:26 smithi076 ceph-mon[24587]: purged_snaps scrub ok 2024-09-13T23:03:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:26 smithi076 ceph-mon[24587]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T23:03:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:26 smithi076 ceph-mon[24587]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T23:03:27.039 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T23:03:27.360 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: Detected new or changed devices on smithi076 2024-09-13T23:03:27.360 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:27.361 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:27.361 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:27.361 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:27.361 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:27.361 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: Adjusting osd_memory_target on smithi076 to 4272M 2024-09-13T23:03:27.361 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:27.361 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:27.361 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:27.361 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:27.361 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:27.361 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:27.361 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[23758]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: Detected new or changed devices on smithi076 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: Adjusting osd_memory_target on smithi076 to 4272M 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:27.362 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:27.363 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:27 smithi022 ceph-mon[28440]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T23:03:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: Detected new or changed devices on smithi076 2024-09-13T23:03:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:03:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: Adjusting osd_memory_target on smithi076 to 4272M 2024-09-13T23:03:27.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:27.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:27.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:27.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:27.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:27.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:27.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:27 smithi076 ceph-mon[24587]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T23:03:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:28 smithi022 ceph-mon[28440]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:28 smithi022 ceph-mon[23758]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:28.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:28 smithi076 ceph-mon[24587]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:29.258 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:03:29.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:29 smithi022 ceph-mon[23758]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T23:03:29.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:29 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/278381305' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T23:03:29.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:29 smithi022 ceph-mon[28440]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T23:03:29.565 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:29 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/278381305' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T23:03:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:29 smithi076 ceph-mon[24587]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T23:03:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:29 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/278381305' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T23:03:29.998 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1726268605,"num_in_osds":8,"osd_in_since":1726268587,"num_remapped_pgs":0} 2024-09-13T23:03:29.998 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T23:03:30.211 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:03:30.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:30 smithi022 ceph-mon[23758]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:30.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:30 smithi022 ceph-mon[23758]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T23:03:30.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:30 smithi022 ceph-mon[28440]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:30.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:30 smithi022 ceph-mon[28440]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T23:03:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:30 smithi076 ceph-mon[24587]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:30 smithi076 ceph-mon[24587]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T23:03:32.130 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:03:32.130 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":59,"fsid":"9adcac8c-7223-11ef-bceb-c7b262605968","created":"2024-09-13T22:58:27.106170+0000","modified":"2024-09-13T23:03:29.304313+0000","last_up_change":"2024-09-13T23:03:25.227963+0000","last_in_change":"2024-09-13T23:03:07.790110+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T23:01:29.128742+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cf3e22af-e2e1-4b53-902a-31f06def8887","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":473621849},{"type":"v1","addr":"172.21.15.22:6803","nonce":473621849}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":473621849},{"type":"v1","addr":"172.21.15.22:6805","nonce":473621849}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":473621849},{"type":"v1","addr":"172.21.15.22:6809","nonce":473621849}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":473621849},{"type":"v1","addr":"172.21.15.22:6807","nonce":473621849}]},"public_addr":"172.21.15.22:6803/473621849","cluster_addr":"172.21.15.22:6805/473621849","heartbeat_back_addr":"172.21.15.22:6809/473621849","heartbeat_front_addr":"172.21.15.22:6807/473621849","state":["exists","up"]},{"osd":1,"uuid":"cb4c98bf-8fbc-4d9c-8141-598fb4168065","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":1475707463},{"type":"v1","addr":"172.21.15.22:6811","nonce":1475707463}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":1475707463},{"type":"v1","addr":"172.21.15.22:6813","nonce":1475707463}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":1475707463},{"type":"v1","addr":"172.21.15.22:6817","nonce":1475707463}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":1475707463},{"type":"v1","addr":"172.21.15.22:6815","nonce":1475707463}]},"public_addr":"172.21.15.22:6811/1475707463","cluster_addr":"172.21.15.22:6813/1475707463","heartbeat_back_addr":"172.21.15.22:6817/1475707463","heartbeat_front_addr":"172.21.15.22:6815/1475707463","state":["exists","up"]},{"osd":2,"uuid":"e5b61686-bddb-4564-bc89-0da23cf64582","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":3250575233},{"type":"v1","addr":"172.21.15.22:6819","nonce":3250575233}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":3250575233},{"type":"v1","addr":"172.21.15.22:6821","nonce":3250575233}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":3250575233},{"type":"v1","addr":"172.21.15.22:6825","nonce":3250575233}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":3250575233},{"type":"v1","addr":"172.21.15.22:6823","nonce":3250575233}]},"public_addr":"172.21.15.22:6819/3250575233","cluster_addr":"172.21.15.22:6821/3250575233","heartbeat_back_addr":"172.21.15.22:6825/3250575233","heartbeat_front_addr":"172.21.15.22:6823/3250575233","state":["exists","up"]},{"osd":3,"uuid":"f5bf14cf-4376-4c35-886e-1a6217af0b8d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":2701159141},{"type":"v1","addr":"172.21.15.22:6827","nonce":2701159141}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":2701159141},{"type":"v1","addr":"172.21.15.22:6829","nonce":2701159141}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":2701159141},{"type":"v1","addr":"172.21.15.22:6833","nonce":2701159141}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":2701159141},{"type":"v1","addr":"172.21.15.22:6831","nonce":2701159141}]},"public_addr":"172.21.15.22:6827/2701159141","cluster_addr":"172.21.15.22:6829/2701159141","heartbeat_back_addr":"172.21.15.22:6833/2701159141","heartbeat_front_addr":"172.21.15.22:6831/2701159141","state":["exists","up"]},{"osd":4,"uuid":"f261f4b3-5255-4944-89ef-17542fcf62d3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6800","nonce":3477942838},{"type":"v1","addr":"172.21.15.76:6801","nonce":3477942838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":3477942838},{"type":"v1","addr":"172.21.15.76:6803","nonce":3477942838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":3477942838},{"type":"v1","addr":"172.21.15.76:6807","nonce":3477942838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":3477942838},{"type":"v1","addr":"172.21.15.76:6805","nonce":3477942838}]},"public_addr":"172.21.15.76:6801/3477942838","cluster_addr":"172.21.15.76:6803/3477942838","heartbeat_back_addr":"172.21.15.76:6807/3477942838","heartbeat_front_addr":"172.21.15.76:6805/3477942838","state":["exists","up"]},{"osd":5,"uuid":"b53bd7f5-fc30-4e46-8deb-43ab8f0c10c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":2219212499},{"type":"v1","addr":"172.21.15.76:6809","nonce":2219212499}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":2219212499},{"type":"v1","addr":"172.21.15.76:6811","nonce":2219212499}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":2219212499},{"type":"v1","addr":"172.21.15.76:6815","nonce":2219212499}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":2219212499},{"type":"v1","addr":"172.21.15.76:6813","nonce":2219212499}]},"public_addr":"172.21.15.76:6809/2219212499","cluster_addr":"172.21.15.76:6811/2219212499","heartbeat_back_addr":"172.21.15.76:6815/2219212499","heartbeat_front_addr":"172.21.15.76:6813/2219212499","state":["exists","up"]},{"osd":6,"uuid":"8901b439-aa14-481e-a3f1-341fe4c467e9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":1379127534},{"type":"v1","addr":"172.21.15.76:6817","nonce":1379127534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":1379127534},{"type":"v1","addr":"172.21.15.76:6819","nonce":1379127534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":1379127534},{"type":"v1","addr":"172.21.15.76:6823","nonce":1379127534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":1379127534},{"type":"v1","addr":"172.21.15.76:6821","nonce":1379127534}]},"public_addr":"172.21.15.76:6817/1379127534","cluster_addr":"172.21.15.76:6819/1379127534","heartbeat_back_addr":"172.21.15.76:6823/1379127534","heartbeat_front_addr":"172.21.15.76:6821/1379127534","state":["exists","up"]},{"osd":7,"uuid":"ad6c9aa3-0472-4806-b96c-161a1cb9005b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":2500464355},{"type":"v1","addr":"172.21.15.76:6825","nonce":2500464355}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":2500464355},{"type":"v1","addr":"172.21.15.76:6827","nonce":2500464355}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":2500464355},{"type":"v1","addr":"172.21.15.76:6831","nonce":2500464355}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":2500464355},{"type":"v1","addr":"172.21.15.76:6829","nonce":2500464355}]},"public_addr":"172.21.15.76:6825/2500464355","cluster_addr":"172.21.15.76:6827/2500464355","heartbeat_back_addr":"172.21.15.76:6831/2500464355","heartbeat_front_addr":"172.21.15.76:6829/2500464355","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:00:36.088060+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:01:00.425297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:01:25.157738+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:01:50.375208+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:02:11.774722+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:02:34.980038+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:02:59.118687+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:03:23.284628+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:0/777163445":"2024-09-14T22:59:14.574157+0000","172.21.15.22:0/4186392471":"2024-09-14T22:59:14.574157+0000","172.21.15.22:6801/3032931192":"2024-09-14T22:59:14.574157+0000","172.21.15.22:6800/3032931192":"2024-09-14T22:59:14.574157+0000","172.21.15.22:0/898169442":"2024-09-14T22:59:14.574157+0000","172.21.15.22:6801/137268489":"2024-09-14T22:58:51.068764+0000","172.21.15.22:0/1254370233":"2024-09-14T22:58:51.068764+0000","172.21.15.22:0/1724895572":"2024-09-14T22:58:51.068764+0000","172.21.15.22:6800/137268489":"2024-09-14T22:58:51.068764+0000","172.21.15.22:0/1746885615":"2024-09-14T22:58:51.068764+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T23:03:32.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:32 smithi022 ceph-mon[23758]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-09-13T23:03:32.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:32 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/647598269' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T23:03:32.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:32 smithi022 ceph-mon[28440]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-09-13T23:03:32.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:32 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/647598269' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T23:03:32.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:32 smithi076 ceph-mon[24587]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-09-13T23:03:32.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:32 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/647598269' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T23:03:33.009 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-13T23:01:29.128742+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-13T23:03:33.010 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-13T23:03:33.231 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:03:34.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:34 smithi022 ceph-mon[28440]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T23:03:34.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:34 smithi022 ceph-mon[23758]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T23:03:34.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:34 smithi076 ceph-mon[24587]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T23:03:35.293 INFO:teuthology.orchestra.run.smithi022.stdout:pg_num: 1 2024-09-13T23:03:35.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:35 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3037019890' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T23:03:35.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:35 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3037019890' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T23:03:35.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:35 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3037019890' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T23:03:36.091 INFO:tasks.cephadm:Adding prometheus.a on smithi076 2024-09-13T23:03:36.091 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi076=a' 2024-09-13T23:03:36.410 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:36 smithi076 ceph-mon[24587]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-13T23:03:36.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:36 smithi022 ceph-mon[23758]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-13T23:03:36.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:36 smithi022 ceph-mon[28440]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-13T23:03:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:37 smithi076 ceph-mon[24587]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-09-13T23:03:38.086 INFO:teuthology.orchestra.run.smithi076.stdout:Scheduled prometheus update... 2024-09-13T23:03:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:37 smithi022 ceph-mon[23758]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-09-13T23:03:38.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:37 smithi022 ceph-mon[28440]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-09-13T23:03:38.746 DEBUG:teuthology.orchestra.run.smithi076:prometheus.a> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-13T23:03:38.748 INFO:tasks.cephadm:Adding node-exporter.a on smithi022 2024-09-13T23:03:38.749 INFO:tasks.cephadm:Adding node-exporter.b on smithi076 2024-09-13T23:03:38.749 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi022=a;smithi076=b' 2024-09-13T23:03:39.348 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-mon[24587]: from='client.34331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi076=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:03:39.348 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-mon[24587]: Saving service prometheus spec with placement smithi076=a;count:1 2024-09-13T23:03:39.348 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:39.348 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:39.348 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:39.349 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:39.349 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:39.349 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T23:03:39.349 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ignoring --setuser ceph since I am not root 2024-09-13T23:03:39.349 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ignoring --setgroup ceph since I am not root 2024-09-13T23:03:39.349 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-mgr[25753]: -- 172.21.15.76:0/3746507363 <== mon.2 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fc139d2340 con 0x55fc139ab000 2024-09-13T23:03:39.349 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:39.263+0000 7f76c311b000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T23:03:39.349 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:39.346+0000 7f76c311b000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T23:03:39.376 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[23758]: from='client.34331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi076=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:03:39.376 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[23758]: Saving service prometheus spec with placement smithi076=a;count:1 2024-09-13T23:03:39.376 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:39.376 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:39.376 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:39.376 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:39.376 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:39.376 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T23:03:39.377 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ignoring --setuser ceph since I am not root 2024-09-13T23:03:39.377 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ignoring --setgroup ceph since I am not root 2024-09-13T23:03:39.377 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mgr[23990]: -- 172.21.15.22:0/3815911153 <== mon.1 v2:172.21.15.22:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56033db8c340 con 0x56033db64800 2024-09-13T23:03:39.377 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:39.290+0000 7f2fee05d000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T23:03:39.377 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:39.373+0000 7f2fee05d000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T23:03:39.377 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[28440]: from='client.34331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi076=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:03:39.377 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[28440]: Saving service prometheus spec with placement smithi076=a;count:1 2024-09-13T23:03:39.377 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:39.377 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:39.378 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:39.378 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:39.378 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' 2024-09-13T23:03:39.378 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T23:03:39.613 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:39.523+0000 7f76c311b000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T23:03:39.664 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:39.589+0000 7f2fee05d000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T23:03:39.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:39.696+0000 7f2fee05d000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T23:03:39.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:39 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:39.835+0000 7f2fee05d000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:03:39.960 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:39.611+0000 7f76c311b000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T23:03:39.960 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:39.745+0000 7f76c311b000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:03:40.277 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:40 smithi076 ceph-mon[24587]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T23:03:40.277 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:40 smithi076 ceph-mon[24587]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T23:03:40.278 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:39.960+0000 7f76c311b000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T23:03:40.402 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:40 smithi022 ceph-mon[23758]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T23:03:40.403 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:40 smithi022 ceph-mon[23758]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T23:03:40.403 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:40 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:40.056+0000 7f2fee05d000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T23:03:40.403 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:40 smithi022 ceph-mon[28440]: from='mgr.14150 172.21.15.22:0/4108267879' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T23:03:40.403 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:40 smithi022 ceph-mon[28440]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T23:03:40.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:40 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:40.313+0000 7f76c311b000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T23:03:40.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:40 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:40.403+0000 7f76c311b000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T23:03:40.667 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:40 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:40.401+0000 7f2fee05d000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T23:03:40.667 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:40 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:40.494+0000 7f2fee05d000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T23:03:40.667 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:40 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:40.665+0000 7f2fee05d000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T23:03:41.036 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:40 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:40.577+0000 7f76c311b000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T23:03:42.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:41.637+0000 7f76c311b000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T23:03:42.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:41.728+0000 7f76c311b000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T23:03:42.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:41 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:41.765+0000 7f2fee05d000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T23:03:42.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:41 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:41.859+0000 7f2fee05d000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T23:03:42.948 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:42.644+0000 7f76c311b000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T23:03:43.123 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:42.803+0000 7f2fee05d000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T23:03:43.268 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:42.948+0000 7f76c311b000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T23:03:43.268 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:43.081+0000 7f76c311b000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:03:43.268 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:43.169+0000 7f76c311b000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T23:03:43.268 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:43.267+0000 7f76c311b000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T23:03:43.426 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:43.122+0000 7f2fee05d000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T23:03:43.426 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:43.260+0000 7f2fee05d000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:03:43.426 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:43.352+0000 7f2fee05d000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T23:03:43.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:43.356+0000 7f76c311b000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T23:03:43.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:43.506+0000 7f76c311b000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T23:03:43.799 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:43.455+0000 7f2fee05d000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T23:03:43.799 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:43.546+0000 7f2fee05d000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T23:03:43.799 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:43.700+0000 7f2fee05d000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T23:03:43.799 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:43.797+0000 7f2fee05d000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T23:03:43.983 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:43.602+0000 7f76c311b000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T23:03:43.983 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:43.701+0000 7f76c311b000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T23:03:43.983 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:43.982+0000 7f76c311b000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T23:03:44.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:43.898+0000 7f2fee05d000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T23:03:44.286 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:44 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:44.072+0000 7f76c311b000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T23:03:44.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:44 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:44.163+0000 7f76c311b000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T23:03:44.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:44 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:44.194+0000 7f2fee05d000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T23:03:44.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:44 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:44.285+0000 7f2fee05d000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T23:03:44.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:44 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:44.377+0000 7f2fee05d000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T23:03:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-mon[24587]: Standby manager daemon x restarted 2024-09-13T23:03:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-mon[24587]: Standby manager daemon x started 2024-09-13T23:03:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/1699946930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:03:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/1699946930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:03:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/1699946930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:03:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/1699946930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:03:45.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:45.236+0000 7f76c311b000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T23:03:45.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: [13/Sep/2024:23:03:45] ENGINE Bus STARTING 2024-09-13T23:03:45.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: CherryPy Checker: 2024-09-13T23:03:45.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: The Application mounted at '' has an empty config. 2024-09-13T23:03:45.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: 2024-09-13T23:03:45.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: [13/Sep/2024:23:03:45] ENGINE Serving on http://:::9283 2024-09-13T23:03:45.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:03:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: [13/Sep/2024:23:03:45] ENGINE Bus STARTED 2024-09-13T23:03:45.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:45.494+0000 7f2fee05d000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T23:03:45.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-mon[23758]: Standby manager daemon x restarted 2024-09-13T23:03:45.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-mon[23758]: Standby manager daemon x started 2024-09-13T23:03:45.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/1699946930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:03:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/1699946930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:03:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/1699946930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:03:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/1699946930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:03:45.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-mon[28440]: Standby manager daemon x restarted 2024-09-13T23:03:45.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-mon[28440]: Standby manager daemon x started 2024-09-13T23:03:45.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/1699946930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:03:45.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/1699946930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:03:45.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/1699946930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:03:45.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:45 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/1699946930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:03:46.560 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:46 smithi076 ceph-mon[24587]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-13T23:03:46.560 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:46 smithi076 ceph-mon[24587]: Active manager daemon y restarted 2024-09-13T23:03:46.560 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:46 smithi076 ceph-mon[24587]: Activating manager daemon y 2024-09-13T23:03:46.561 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:46 smithi076 ceph-mon[24587]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T23:03:46.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-mon[28440]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-13T23:03:46.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-mon[28440]: Active manager daemon y restarted 2024-09-13T23:03:46.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-mon[28440]: Activating manager daemon y 2024-09-13T23:03:46.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-mon[28440]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T23:03:46.637 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-mon[23758]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-13T23:03:46.637 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-mon[23758]: Active manager daemon y restarted 2024-09-13T23:03:46.637 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-mon[23758]: Activating manager daemon y 2024-09-13T23:03:46.637 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-mon[23758]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T23:03:46.638 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:23:03:46] ENGINE Bus STARTING 2024-09-13T23:03:46.638 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: CherryPy Checker: 2024-09-13T23:03:46.638 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: The Application mounted at '' has an empty config. 2024-09-13T23:03:46.638 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:03:46.638 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:23:03:46] ENGINE Serving on http://:::9283 2024-09-13T23:03:46.638 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:46 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:23:03:46] ENGINE Bus STARTED 2024-09-13T23:03:47.332 INFO:teuthology.orchestra.run.smithi076.stdout:Scheduled node-exporter update... 2024-09-13T23:03:47.494 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: mgrmap e18: y(active, starting, since 0.806683s), standbys: x 2024-09-13T23:03:47.494 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:03:47.494 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:03:47.494 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:03:47.494 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:03:47.494 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:03:47.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:03:47.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:03:47.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:03:47.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:03:47.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:03:47.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:03:47.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:47.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:47.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:03:47.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:03:47.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:03:47.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: Manager daemon y is now available 2024-09-13T23:03:47.496 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:47.496 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:47.496 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:47.496 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:47.496 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:47.496 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:03:47.496 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:03:47.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: mgrmap e18: y(active, starting, since 0.806683s), standbys: x 2024-09-13T23:03:47.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:03:47.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:03:47.497 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:03:47.498 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:03:47.498 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: Manager daemon y is now available 2024-09-13T23:03:47.498 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:47.498 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:47.498 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:47.498 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:47.498 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:47.498 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:03:47.498 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:03:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: mgrmap e18: y(active, starting, since 0.806683s), standbys: x 2024-09-13T23:03:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:03:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:03:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:03:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:03:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:03:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:03:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:03:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:03:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:03:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:03:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:03:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:03:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:03:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:03:47.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:03:47.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:03:47.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: Manager daemon y is now available 2024-09-13T23:03:47.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:47.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:03:47.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:03:47.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:03:47.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:47.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:03:47.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:03:47.871 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:23:03:47] ENGINE Bus STARTING 2024-09-13T23:03:47.871 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:23:03:47] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:03:47.871 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:03:47 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:23:03:47] ENGINE Bus STARTED 2024-09-13T23:03:48.182 DEBUG:teuthology.orchestra.run.smithi022:node-exporter.a> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-13T23:03:48.184 DEBUG:teuthology.orchestra.run.smithi076:node-exporter.b> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-13T23:03:48.187 INFO:tasks.cephadm:Adding alertmanager.a on smithi022 2024-09-13T23:03:48.187 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi022=a' 2024-09-13T23:03:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[23758]: mgrmap e19: y(active, since 1.82303s), standbys: x 2024-09-13T23:03:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[23758]: Saving service node-exporter spec with placement smithi022=a;smithi076=b;count:2 2024-09-13T23:03:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[23758]: [13/Sep/2024:23:03:47] ENGINE Bus STARTING 2024-09-13T23:03:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[23758]: [13/Sep/2024:23:03:47] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:03:48.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[23758]: [13/Sep/2024:23:03:47] ENGINE Bus STARTED 2024-09-13T23:03:48.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:48.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[28440]: mgrmap e19: y(active, since 1.82303s), standbys: x 2024-09-13T23:03:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[28440]: Saving service node-exporter spec with placement smithi022=a;smithi076=b;count:2 2024-09-13T23:03:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[28440]: [13/Sep/2024:23:03:47] ENGINE Bus STARTING 2024-09-13T23:03:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[28440]: [13/Sep/2024:23:03:47] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:03:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[28440]: [13/Sep/2024:23:03:47] ENGINE Bus STARTED 2024-09-13T23:03:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:48 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:48.431 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:48 smithi076 ceph-mon[24587]: mgrmap e19: y(active, since 1.82303s), standbys: x 2024-09-13T23:03:48.431 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:48 smithi076 ceph-mon[24587]: Saving service node-exporter spec with placement smithi022=a;smithi076=b;count:2 2024-09-13T23:03:48.431 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:48 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:48.431 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:48 smithi076 ceph-mon[24587]: [13/Sep/2024:23:03:47] ENGINE Bus STARTING 2024-09-13T23:03:48.431 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:48 smithi076 ceph-mon[24587]: [13/Sep/2024:23:03:47] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:03:48.431 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:48 smithi076 ceph-mon[24587]: [13/Sep/2024:23:03:47] ENGINE Bus STARTED 2024-09-13T23:03:48.431 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:48 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:48.431 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:48 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:49.416 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:49 smithi076 ceph-mon[24587]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:49.416 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:49 smithi076 ceph-mon[24587]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-13T23:03:49.416 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:49 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:49.417 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:49 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:49.417 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:49 smithi076 ceph-mon[24587]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:03:49.417 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:49 smithi076 ceph-mon[24587]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:03:49.417 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:49 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:49.417 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:49 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:49.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[23758]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:49.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[23758]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-13T23:03:49.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:49.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:49.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[23758]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:03:49.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[23758]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:03:49.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:49.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:49.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[28440]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:49.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[28440]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-13T23:03:49.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:49.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:49.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:03:49.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:03:49.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:49.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:49 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:50.372 INFO:teuthology.orchestra.run.smithi076.stdout:Scheduled alertmanager update... 2024-09-13T23:03:50.676 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:50 smithi076 ceph-mon[24587]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:03:50.676 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:50 smithi076 ceph-mon[24587]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:03:50.676 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:50 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:50.676 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:50 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:50.676 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:50 smithi076 ceph-mon[24587]: Deploying daemon prometheus.a on smithi076 2024-09-13T23:03:50.676 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:50 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:50.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:50 smithi022 ceph-mon[23758]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:03:50.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:50 smithi022 ceph-mon[23758]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:03:50.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:50 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:50.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:50 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:50.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:50 smithi022 ceph-mon[23758]: Deploying daemon prometheus.a on smithi076 2024-09-13T23:03:50.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:50 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:50.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:50 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:03:50.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:50 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:03:50.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:50 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:50.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:50 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:50.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:50 smithi022 ceph-mon[28440]: Deploying daemon prometheus.a on smithi076 2024-09-13T23:03:50.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:50 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:51.142 DEBUG:teuthology.orchestra.run.smithi022:alertmanager.a> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-13T23:03:51.145 INFO:tasks.cephadm:Adding grafana.a on smithi076 2024-09-13T23:03:51.145 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi076=a' 2024-09-13T23:03:51.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:51 smithi076 ceph-mon[24587]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:51.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:51 smithi076 ceph-mon[24587]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi022=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:03:51.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:51 smithi076 ceph-mon[24587]: Saving service alertmanager spec with placement smithi022=a;count:1 2024-09-13T23:03:51.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:51 smithi022 ceph-mon[23758]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:51.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:51 smithi022 ceph-mon[23758]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi022=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:03:51.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:51 smithi022 ceph-mon[23758]: Saving service alertmanager spec with placement smithi022=a;count:1 2024-09-13T23:03:51.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:51 smithi022 ceph-mon[28440]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:51.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:51 smithi022 ceph-mon[28440]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi022=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:03:51.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:51 smithi022 ceph-mon[28440]: Saving service alertmanager spec with placement smithi022=a;count:1 2024-09-13T23:03:54.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:53 smithi076 ceph-mon[24587]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:54.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:53 smithi022 ceph-mon[23758]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:54.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:53 smithi022 ceph-mon[28440]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:54.958 INFO:teuthology.orchestra.run.smithi076.stdout:Scheduled grafana update... 2024-09-13T23:03:56.145 DEBUG:teuthology.orchestra.run.smithi076:grafana.a> sudo journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@grafana.a.service 2024-09-13T23:03:56.148 INFO:tasks.cephadm:Setting up client nodes... 2024-09-13T23:03:56.148 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T23:03:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:55 smithi076 ceph-mon[24587]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:55 smithi076 ceph-mon[24587]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi076=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:03:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:55 smithi076 ceph-mon[24587]: Saving service grafana spec with placement smithi076=a;count:1 2024-09-13T23:03:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:55 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:56.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:55 smithi022 ceph-mon[23758]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:56.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:55 smithi022 ceph-mon[23758]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi076=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:03:56.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:55 smithi022 ceph-mon[23758]: Saving service grafana spec with placement smithi076=a;count:1 2024-09-13T23:03:56.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:55 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:55 smithi022 ceph-mon[28440]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:55 smithi022 ceph-mon[28440]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi076=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:03:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:55 smithi022 ceph-mon[28440]: Saving service grafana spec with placement smithi076=a;count:1 2024-09-13T23:03:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:55 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:03:58.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:57 smithi022 ceph-mon[23758]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:58.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:57 smithi022 ceph-mon[28440]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:57 smithi076 ceph-mon[24587]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:03:58.570 INFO:teuthology.orchestra.run.smithi022.stdout:[client.0] 2024-09-13T23:03:58.570 INFO:teuthology.orchestra.run.smithi022.stdout: key = AQDexORmUhO6IRAACmCTvrF+h5NbHYjs0HCG4Q== 2024-09-13T23:03:58.964 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:58 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1697572251' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T23:03:58.964 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:58 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T23:03:59.156 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:58 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1697572251' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T23:03:59.156 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:58 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T23:03:59.156 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:58 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T23:03:59.157 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:58 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1697572251' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T23:03:59.157 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:58 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T23:03:59.157 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:58 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T23:03:59.179 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-13T23:03:59.179 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-13T23:03:59.180 DEBUG:teuthology.orchestra.run.smithi022:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-13T23:03:59.220 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T23:03:59.247 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:58 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T23:04:00.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:03:59 smithi022 ceph-mon[23758]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:00.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:03:59 smithi022 ceph-mon[28440]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:03:59 smithi076 ceph-mon[24587]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:02.031 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:01 smithi076 ceph-mon[24587]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:02.097 INFO:teuthology.orchestra.run.smithi076.stdout:[client.1] 2024-09-13T23:04:02.098 INFO:teuthology.orchestra.run.smithi076.stdout: key = AQDixORmEs2RBRAAr2Ri1kOjvnr9G9b1Fs6exw== 2024-09-13T23:04:02.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:01 smithi022 ceph-mon[23758]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:02.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:01 smithi022 ceph-mon[28440]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:03.033 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-13T23:04:03.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-13T23:04:03.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-13T23:04:03.112 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T23:04:03.112 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T23:04:03.112 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T23:04:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:02 smithi076 ceph-mon[24587]: from='client.? 172.21.15.76:0/3850670567' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T23:04:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:02 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T23:04:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:02 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T23:04:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:02 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:03.324 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:03 smithi022 systemd[1]: Starting Ceph node-exporter.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:04:03.324 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:02 smithi022 ceph-mon[23758]: from='client.? 172.21.15.76:0/3850670567' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T23:04:03.325 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:02 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T23:04:03.325 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:02 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T23:04:03.325 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:02 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:03.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:02 smithi022 ceph-mon[28440]: from='client.? 172.21.15.76:0/3850670567' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T23:04:03.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:02 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T23:04:03.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:02 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T23:04:03.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:02 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:03.341 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:04.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:04 smithi022 ceph-mon[23758]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:04.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:04 smithi022 ceph-mon[23758]: Deploying daemon node-exporter.a on smithi022 2024-09-13T23:04:04.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:04 smithi022 ceph-mon[28440]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:04.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:04 smithi022 ceph-mon[28440]: Deploying daemon node-exporter.a on smithi022 2024-09-13T23:04:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:04 smithi076 ceph-mon[24587]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:04 smithi076 ceph-mon[24587]: Deploying daemon node-exporter.a on smithi022 2024-09-13T23:04:04.892 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:04 smithi022 bash[43220]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-09-13T23:04:05.176 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:05 smithi022 bash[43220]: Getting image source signatures 2024-09-13T23:04:05.176 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:05 smithi022 bash[43220]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-09-13T23:04:05.176 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:05 smithi022 bash[43220]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-09-13T23:04:05.177 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:05 smithi022 bash[43220]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-09-13T23:04:05.397 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:04:06.364 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:06 smithi022 ceph-mon[23758]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:06.364 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:06 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3336794582' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T23:04:06.365 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:06 smithi022 ceph-mon[28440]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:06.365 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:06 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3336794582' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T23:04:06.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:06 smithi076 ceph-mon[24587]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:06.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:06 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3336794582' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T23:04:06.918 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":20,"active_gid":24352,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6800","nonce":3501087460},{"type":"v1","addr":"172.21.15.22:6801","nonce":3501087460}]},"active_addr":"172.21.15.22:6801/3501087460","active_change":"2024-09-13T23:03:45.498238+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":34337,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.22:8443/","prometheus":"http://172.21.15.22:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":1179215007}]},{"addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":2824990478}]},{"addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":3375084674}]},{"addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":282633179}]}]}} 2024-09-13T23:04:06.922 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T23:04:06.922 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T23:04:06.922 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T23:04:07.035 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:06 smithi022 bash[43220]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-09-13T23:04:07.035 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 bash[43220]: Writing manifest to image destination 2024-09-13T23:04:07.297 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:07.625 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-mon[23758]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:07.625 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:07.626 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-mon[28440]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:07.626 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:07.626 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 podman[43220]: 2024-09-13 23:04:07.29039122 +0000 UTC m=+3.099847802 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-09-13T23:04:07.626 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 podman[43220]: 2024-09-13 23:04:07.397361246 +0000 UTC m=+3.206824105 container create 68ffcaba64d77df95582b553f6fa61fea9d69ee25e29a0c5ef3d5cf04a8aee22 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T23:04:07.626 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 podman[43220]: 2024-09-13 23:04:07.624949982 +0000 UTC m=+3.434406563 container init 68ffcaba64d77df95582b553f6fa61fea9d69ee25e29a0c5ef3d5cf04a8aee22 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T23:04:07.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:07 smithi076 ceph-mon[24587]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:07.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:07 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:07.876 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 podman[43220]: 2024-09-13 23:04:07.629105203 +0000 UTC m=+3.438561785 container start 68ffcaba64d77df95582b553f6fa61fea9d69ee25e29a0c5ef3d5cf04a8aee22 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T23:04:07.876 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.632Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-09-13T23:04:07.876 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.633Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-09-13T23:04:07.876 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.633Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-09-13T23:04:07.876 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.633Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T23:04:07.876 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-09-13T23:04:07.876 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=arp 2024-09-13T23:04:07.877 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=bcache 2024-09-13T23:04:07.877 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=bonding 2024-09-13T23:04:07.877 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=btrfs 2024-09-13T23:04:07.877 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=conntrack 2024-09-13T23:04:07.877 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=cpu 2024-09-13T23:04:07.877 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-09-13T23:04:07.877 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=diskstats 2024-09-13T23:04:07.877 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=dmi 2024-09-13T23:04:07.877 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=edac 2024-09-13T23:04:07.877 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=entropy 2024-09-13T23:04:07.877 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-09-13T23:04:07.878 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=filefd 2024-09-13T23:04:07.878 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=filesystem 2024-09-13T23:04:07.878 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=hwmon 2024-09-13T23:04:07.878 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=infiniband 2024-09-13T23:04:07.878 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=ipvs 2024-09-13T23:04:07.878 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=loadavg 2024-09-13T23:04:07.878 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=mdadm 2024-09-13T23:04:07.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=meminfo 2024-09-13T23:04:07.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=netclass 2024-09-13T23:04:07.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=netdev 2024-09-13T23:04:07.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=netstat 2024-09-13T23:04:07.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=nfs 2024-09-13T23:04:07.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=nfsd 2024-09-13T23:04:07.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=nvme 2024-09-13T23:04:07.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=os 2024-09-13T23:04:07.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-09-13T23:04:07.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=pressure 2024-09-13T23:04:07.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=rapl 2024-09-13T23:04:07.880 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=schedstat 2024-09-13T23:04:07.880 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=sockstat 2024-09-13T23:04:07.880 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=softnet 2024-09-13T23:04:07.880 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=stat 2024-09-13T23:04:07.880 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=tapestats 2024-09-13T23:04:07.880 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=textfile 2024-09-13T23:04:07.880 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-09-13T23:04:07.880 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=time 2024-09-13T23:04:07.880 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-09-13T23:04:07.880 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=uname 2024-09-13T23:04:07.880 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=vmstat 2024-09-13T23:04:07.881 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=xfs 2024-09-13T23:04:07.881 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:115 level=info collector=zfs 2024-09-13T23:04:07.881 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-09-13T23:04:07.881 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[43411]: ts=2024-09-13T23:04:07.634Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-09-13T23:04:07.881 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 bash[43220]: 68ffcaba64d77df95582b553f6fa61fea9d69ee25e29a0c5ef3d5cf04a8aee22 2024-09-13T23:04:07.881 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:04:07 smithi022 systemd[1]: Started Ceph node-exporter.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:04:08.080 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:04:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:04:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:04:08.787 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:08 smithi076 bash[39934]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-09-13T23:04:08.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:08 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:08.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:08 smithi022 ceph-mon[23758]: Deploying daemon node-exporter.b on smithi076 2024-09-13T23:04:08.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:08 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:08.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:08 smithi022 ceph-mon[28440]: Deploying daemon node-exporter.b on smithi076 2024-09-13T23:04:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:08 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:08 smithi076 ceph-mon[24587]: Deploying daemon node-exporter.b on smithi076 2024-09-13T23:04:09.287 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:08 smithi076 bash[39934]: Getting image source signatures 2024-09-13T23:04:09.287 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:08 smithi076 bash[39934]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-09-13T23:04:09.287 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:08 smithi076 bash[39934]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-09-13T23:04:09.287 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:08 smithi076 bash[39934]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-09-13T23:04:09.675 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:04:09.675 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":60,"fsid":"9adcac8c-7223-11ef-bceb-c7b262605968","created":"2024-09-13T22:58:27.106170+0000","modified":"2024-09-13T23:03:45.497909+0000","last_up_change":"2024-09-13T23:03:25.227963+0000","last_in_change":"2024-09-13T23:03:07.790110+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T23:01:29.128742+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cf3e22af-e2e1-4b53-902a-31f06def8887","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":473621849},{"type":"v1","addr":"172.21.15.22:6803","nonce":473621849}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":473621849},{"type":"v1","addr":"172.21.15.22:6805","nonce":473621849}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":473621849},{"type":"v1","addr":"172.21.15.22:6809","nonce":473621849}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":473621849},{"type":"v1","addr":"172.21.15.22:6807","nonce":473621849}]},"public_addr":"172.21.15.22:6803/473621849","cluster_addr":"172.21.15.22:6805/473621849","heartbeat_back_addr":"172.21.15.22:6809/473621849","heartbeat_front_addr":"172.21.15.22:6807/473621849","state":["exists","up"]},{"osd":1,"uuid":"cb4c98bf-8fbc-4d9c-8141-598fb4168065","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":1475707463},{"type":"v1","addr":"172.21.15.22:6811","nonce":1475707463}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":1475707463},{"type":"v1","addr":"172.21.15.22:6813","nonce":1475707463}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":1475707463},{"type":"v1","addr":"172.21.15.22:6817","nonce":1475707463}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":1475707463},{"type":"v1","addr":"172.21.15.22:6815","nonce":1475707463}]},"public_addr":"172.21.15.22:6811/1475707463","cluster_addr":"172.21.15.22:6813/1475707463","heartbeat_back_addr":"172.21.15.22:6817/1475707463","heartbeat_front_addr":"172.21.15.22:6815/1475707463","state":["exists","up"]},{"osd":2,"uuid":"e5b61686-bddb-4564-bc89-0da23cf64582","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":3250575233},{"type":"v1","addr":"172.21.15.22:6819","nonce":3250575233}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":3250575233},{"type":"v1","addr":"172.21.15.22:6821","nonce":3250575233}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":3250575233},{"type":"v1","addr":"172.21.15.22:6825","nonce":3250575233}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":3250575233},{"type":"v1","addr":"172.21.15.22:6823","nonce":3250575233}]},"public_addr":"172.21.15.22:6819/3250575233","cluster_addr":"172.21.15.22:6821/3250575233","heartbeat_back_addr":"172.21.15.22:6825/3250575233","heartbeat_front_addr":"172.21.15.22:6823/3250575233","state":["exists","up"]},{"osd":3,"uuid":"f5bf14cf-4376-4c35-886e-1a6217af0b8d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":2701159141},{"type":"v1","addr":"172.21.15.22:6827","nonce":2701159141}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":2701159141},{"type":"v1","addr":"172.21.15.22:6829","nonce":2701159141}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":2701159141},{"type":"v1","addr":"172.21.15.22:6833","nonce":2701159141}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":2701159141},{"type":"v1","addr":"172.21.15.22:6831","nonce":2701159141}]},"public_addr":"172.21.15.22:6827/2701159141","cluster_addr":"172.21.15.22:6829/2701159141","heartbeat_back_addr":"172.21.15.22:6833/2701159141","heartbeat_front_addr":"172.21.15.22:6831/2701159141","state":["exists","up"]},{"osd":4,"uuid":"f261f4b3-5255-4944-89ef-17542fcf62d3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6800","nonce":3477942838},{"type":"v1","addr":"172.21.15.76:6801","nonce":3477942838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":3477942838},{"type":"v1","addr":"172.21.15.76:6803","nonce":3477942838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":3477942838},{"type":"v1","addr":"172.21.15.76:6807","nonce":3477942838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":3477942838},{"type":"v1","addr":"172.21.15.76:6805","nonce":3477942838}]},"public_addr":"172.21.15.76:6801/3477942838","cluster_addr":"172.21.15.76:6803/3477942838","heartbeat_back_addr":"172.21.15.76:6807/3477942838","heartbeat_front_addr":"172.21.15.76:6805/3477942838","state":["exists","up"]},{"osd":5,"uuid":"b53bd7f5-fc30-4e46-8deb-43ab8f0c10c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":2219212499},{"type":"v1","addr":"172.21.15.76:6809","nonce":2219212499}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":2219212499},{"type":"v1","addr":"172.21.15.76:6811","nonce":2219212499}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":2219212499},{"type":"v1","addr":"172.21.15.76:6815","nonce":2219212499}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":2219212499},{"type":"v1","addr":"172.21.15.76:6813","nonce":2219212499}]},"public_addr":"172.21.15.76:6809/2219212499","cluster_addr":"172.21.15.76:6811/2219212499","heartbeat_back_addr":"172.21.15.76:6815/2219212499","heartbeat_front_addr":"172.21.15.76:6813/2219212499","state":["exists","up"]},{"osd":6,"uuid":"8901b439-aa14-481e-a3f1-341fe4c467e9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":1379127534},{"type":"v1","addr":"172.21.15.76:6817","nonce":1379127534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":1379127534},{"type":"v1","addr":"172.21.15.76:6819","nonce":1379127534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":1379127534},{"type":"v1","addr":"172.21.15.76:6823","nonce":1379127534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":1379127534},{"type":"v1","addr":"172.21.15.76:6821","nonce":1379127534}]},"public_addr":"172.21.15.76:6817/1379127534","cluster_addr":"172.21.15.76:6819/1379127534","heartbeat_back_addr":"172.21.15.76:6823/1379127534","heartbeat_front_addr":"172.21.15.76:6821/1379127534","state":["exists","up"]},{"osd":7,"uuid":"ad6c9aa3-0472-4806-b96c-161a1cb9005b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":2500464355},{"type":"v1","addr":"172.21.15.76:6825","nonce":2500464355}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":2500464355},{"type":"v1","addr":"172.21.15.76:6827","nonce":2500464355}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":2500464355},{"type":"v1","addr":"172.21.15.76:6831","nonce":2500464355}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":2500464355},{"type":"v1","addr":"172.21.15.76:6829","nonce":2500464355}]},"public_addr":"172.21.15.76:6825/2500464355","cluster_addr":"172.21.15.76:6827/2500464355","heartbeat_back_addr":"172.21.15.76:6831/2500464355","heartbeat_front_addr":"172.21.15.76:6829/2500464355","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:00:36.088060+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:01:00.425297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:01:25.157738+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:01:50.375208+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:02:11.774722+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:02:34.980038+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:02:59.118687+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:03:23.284628+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:0/1481920223":"2024-09-14T23:03:45.497872+0000","172.21.15.22:0/2315041159":"2024-09-14T23:03:45.497872+0000","172.21.15.22:0/3245781136":"2024-09-14T23:03:45.497872+0000","172.21.15.22:6801/84729394":"2024-09-14T23:03:45.497872+0000","172.21.15.22:0/777163445":"2024-09-14T22:59:14.574157+0000","172.21.15.22:0/1891390217":"2024-09-14T23:03:45.497872+0000","172.21.15.22:0/4186392471":"2024-09-14T22:59:14.574157+0000","172.21.15.22:6801/3032931192":"2024-09-14T22:59:14.574157+0000","172.21.15.22:6800/3032931192":"2024-09-14T22:59:14.574157+0000","172.21.15.22:6800/84729394":"2024-09-14T23:03:45.497872+0000","172.21.15.22:0/898169442":"2024-09-14T22:59:14.574157+0000","172.21.15.22:6801/137268489":"2024-09-14T22:58:51.068764+0000","172.21.15.22:0/1254370233":"2024-09-14T22:58:51.068764+0000","172.21.15.22:0/1724895572":"2024-09-14T22:58:51.068764+0000","172.21.15.22:6800/137268489":"2024-09-14T22:58:51.068764+0000","172.21.15.22:0/1746885615":"2024-09-14T22:58:51.068764+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T23:04:09.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:09 smithi022 ceph-mon[23758]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:09.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:09 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2575290120' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T23:04:09.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:09 smithi022 ceph-mon[28440]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:09.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:09 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2575290120' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T23:04:10.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:09 smithi076 ceph-mon[24587]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:09 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2575290120' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T23:04:10.571 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T23:04:10.571 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T23:04:10.787 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:10 smithi076 bash[39934]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-09-13T23:04:10.787 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:10 smithi076 bash[39934]: Writing manifest to image destination 2024-09-13T23:04:10.795 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:11.119 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:10 smithi076 podman[39934]: 2024-09-13 23:04:10.818163568 +0000 UTC m=+2.273276558 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-09-13T23:04:11.119 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:10 smithi076 podman[39934]: 2024-09-13 23:04:10.908534853 +0000 UTC m=+2.363647843 container create 6ee1b8ce1803913235dfe6676635a2582db9fd91724d8033273437f7cba5b822 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T23:04:11.120 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 podman[39934]: 2024-09-13 23:04:11.119294395 +0000 UTC m=+2.574407392 container init 6ee1b8ce1803913235dfe6676635a2582db9fd91724d8033273437f7cba5b822 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T23:04:11.537 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 podman[39934]: 2024-09-13 23:04:11.122329547 +0000 UTC m=+2.577442545 container start 6ee1b8ce1803913235dfe6676635a2582db9fd91724d8033273437f7cba5b822 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T23:04:11.538 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.126Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-09-13T23:04:11.538 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.126Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-09-13T23:04:11.538 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.126Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-09-13T23:04:11.538 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.126Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T23:04:11.538 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-09-13T23:04:11.538 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=arp 2024-09-13T23:04:11.538 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=bcache 2024-09-13T23:04:11.538 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=bonding 2024-09-13T23:04:11.538 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=btrfs 2024-09-13T23:04:11.538 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=conntrack 2024-09-13T23:04:11.538 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=cpu 2024-09-13T23:04:11.538 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-09-13T23:04:11.539 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=diskstats 2024-09-13T23:04:11.539 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=dmi 2024-09-13T23:04:11.539 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=edac 2024-09-13T23:04:11.539 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=entropy 2024-09-13T23:04:11.539 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-09-13T23:04:11.539 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=filefd 2024-09-13T23:04:11.539 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=filesystem 2024-09-13T23:04:11.539 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=hwmon 2024-09-13T23:04:11.540 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=infiniband 2024-09-13T23:04:11.540 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=ipvs 2024-09-13T23:04:11.540 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=loadavg 2024-09-13T23:04:11.540 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=mdadm 2024-09-13T23:04:11.540 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=meminfo 2024-09-13T23:04:11.540 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=netclass 2024-09-13T23:04:11.540 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=netdev 2024-09-13T23:04:11.540 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=netstat 2024-09-13T23:04:11.540 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=nfs 2024-09-13T23:04:11.540 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=nfsd 2024-09-13T23:04:11.540 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=nvme 2024-09-13T23:04:11.540 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=os 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=pressure 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=rapl 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=schedstat 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=sockstat 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=softnet 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=stat 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=tapestats 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=textfile 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=time 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-09-13T23:04:11.541 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=uname 2024-09-13T23:04:11.542 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=vmstat 2024-09-13T23:04:11.542 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=xfs 2024-09-13T23:04:11.542 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:115 level=info collector=zfs 2024-09-13T23:04:11.542 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.127Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-09-13T23:04:11.542 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[40008]: ts=2024-09-13T23:04:11.128Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-09-13T23:04:11.542 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 bash[39934]: 6ee1b8ce1803913235dfe6676635a2582db9fd91724d8033273437f7cba5b822 2024-09-13T23:04:11.542 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:04:11 smithi076 systemd[1]: Started Ceph node-exporter.b for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:04:11.873 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:04:11] "GET /metrics HTTP/1.1" 200 191494 "" "Prometheus/2.33.4" 2024-09-13T23:04:11.873 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-mon[28440]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:11.873 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:11.873 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:04:11.873 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:04:11.873 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:04:11.874 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:11.874 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-mon[23758]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:11.874 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:11.874 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:04:11.874 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:04:11.874 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:04:11.874 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:11 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-mon[24587]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:04:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:04:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:04:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:11 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:12.573 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:04:12.574 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":60,"fsid":"9adcac8c-7223-11ef-bceb-c7b262605968","created":"2024-09-13T22:58:27.106170+0000","modified":"2024-09-13T23:03:45.497909+0000","last_up_change":"2024-09-13T23:03:25.227963+0000","last_in_change":"2024-09-13T23:03:07.790110+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T23:01:29.128742+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cf3e22af-e2e1-4b53-902a-31f06def8887","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":473621849},{"type":"v1","addr":"172.21.15.22:6803","nonce":473621849}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":473621849},{"type":"v1","addr":"172.21.15.22:6805","nonce":473621849}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":473621849},{"type":"v1","addr":"172.21.15.22:6809","nonce":473621849}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":473621849},{"type":"v1","addr":"172.21.15.22:6807","nonce":473621849}]},"public_addr":"172.21.15.22:6803/473621849","cluster_addr":"172.21.15.22:6805/473621849","heartbeat_back_addr":"172.21.15.22:6809/473621849","heartbeat_front_addr":"172.21.15.22:6807/473621849","state":["exists","up"]},{"osd":1,"uuid":"cb4c98bf-8fbc-4d9c-8141-598fb4168065","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":1475707463},{"type":"v1","addr":"172.21.15.22:6811","nonce":1475707463}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":1475707463},{"type":"v1","addr":"172.21.15.22:6813","nonce":1475707463}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":1475707463},{"type":"v1","addr":"172.21.15.22:6817","nonce":1475707463}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":1475707463},{"type":"v1","addr":"172.21.15.22:6815","nonce":1475707463}]},"public_addr":"172.21.15.22:6811/1475707463","cluster_addr":"172.21.15.22:6813/1475707463","heartbeat_back_addr":"172.21.15.22:6817/1475707463","heartbeat_front_addr":"172.21.15.22:6815/1475707463","state":["exists","up"]},{"osd":2,"uuid":"e5b61686-bddb-4564-bc89-0da23cf64582","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":3250575233},{"type":"v1","addr":"172.21.15.22:6819","nonce":3250575233}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":3250575233},{"type":"v1","addr":"172.21.15.22:6821","nonce":3250575233}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":3250575233},{"type":"v1","addr":"172.21.15.22:6825","nonce":3250575233}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":3250575233},{"type":"v1","addr":"172.21.15.22:6823","nonce":3250575233}]},"public_addr":"172.21.15.22:6819/3250575233","cluster_addr":"172.21.15.22:6821/3250575233","heartbeat_back_addr":"172.21.15.22:6825/3250575233","heartbeat_front_addr":"172.21.15.22:6823/3250575233","state":["exists","up"]},{"osd":3,"uuid":"f5bf14cf-4376-4c35-886e-1a6217af0b8d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":2701159141},{"type":"v1","addr":"172.21.15.22:6827","nonce":2701159141}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":2701159141},{"type":"v1","addr":"172.21.15.22:6829","nonce":2701159141}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":2701159141},{"type":"v1","addr":"172.21.15.22:6833","nonce":2701159141}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":2701159141},{"type":"v1","addr":"172.21.15.22:6831","nonce":2701159141}]},"public_addr":"172.21.15.22:6827/2701159141","cluster_addr":"172.21.15.22:6829/2701159141","heartbeat_back_addr":"172.21.15.22:6833/2701159141","heartbeat_front_addr":"172.21.15.22:6831/2701159141","state":["exists","up"]},{"osd":4,"uuid":"f261f4b3-5255-4944-89ef-17542fcf62d3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6800","nonce":3477942838},{"type":"v1","addr":"172.21.15.76:6801","nonce":3477942838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":3477942838},{"type":"v1","addr":"172.21.15.76:6803","nonce":3477942838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":3477942838},{"type":"v1","addr":"172.21.15.76:6807","nonce":3477942838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":3477942838},{"type":"v1","addr":"172.21.15.76:6805","nonce":3477942838}]},"public_addr":"172.21.15.76:6801/3477942838","cluster_addr":"172.21.15.76:6803/3477942838","heartbeat_back_addr":"172.21.15.76:6807/3477942838","heartbeat_front_addr":"172.21.15.76:6805/3477942838","state":["exists","up"]},{"osd":5,"uuid":"b53bd7f5-fc30-4e46-8deb-43ab8f0c10c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":2219212499},{"type":"v1","addr":"172.21.15.76:6809","nonce":2219212499}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":2219212499},{"type":"v1","addr":"172.21.15.76:6811","nonce":2219212499}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":2219212499},{"type":"v1","addr":"172.21.15.76:6815","nonce":2219212499}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":2219212499},{"type":"v1","addr":"172.21.15.76:6813","nonce":2219212499}]},"public_addr":"172.21.15.76:6809/2219212499","cluster_addr":"172.21.15.76:6811/2219212499","heartbeat_back_addr":"172.21.15.76:6815/2219212499","heartbeat_front_addr":"172.21.15.76:6813/2219212499","state":["exists","up"]},{"osd":6,"uuid":"8901b439-aa14-481e-a3f1-341fe4c467e9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":1379127534},{"type":"v1","addr":"172.21.15.76:6817","nonce":1379127534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":1379127534},{"type":"v1","addr":"172.21.15.76:6819","nonce":1379127534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":1379127534},{"type":"v1","addr":"172.21.15.76:6823","nonce":1379127534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":1379127534},{"type":"v1","addr":"172.21.15.76:6821","nonce":1379127534}]},"public_addr":"172.21.15.76:6817/1379127534","cluster_addr":"172.21.15.76:6819/1379127534","heartbeat_back_addr":"172.21.15.76:6823/1379127534","heartbeat_front_addr":"172.21.15.76:6821/1379127534","state":["exists","up"]},{"osd":7,"uuid":"ad6c9aa3-0472-4806-b96c-161a1cb9005b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":2500464355},{"type":"v1","addr":"172.21.15.76:6825","nonce":2500464355}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":2500464355},{"type":"v1","addr":"172.21.15.76:6827","nonce":2500464355}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":2500464355},{"type":"v1","addr":"172.21.15.76:6831","nonce":2500464355}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":2500464355},{"type":"v1","addr":"172.21.15.76:6829","nonce":2500464355}]},"public_addr":"172.21.15.76:6825/2500464355","cluster_addr":"172.21.15.76:6827/2500464355","heartbeat_back_addr":"172.21.15.76:6831/2500464355","heartbeat_front_addr":"172.21.15.76:6829/2500464355","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:00:36.088060+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:01:00.425297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:01:25.157738+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:01:50.375208+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:02:11.774722+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:02:34.980038+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:02:59.118687+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T23:03:23.284628+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:0/1481920223":"2024-09-14T23:03:45.497872+0000","172.21.15.22:0/2315041159":"2024-09-14T23:03:45.497872+0000","172.21.15.22:0/3245781136":"2024-09-14T23:03:45.497872+0000","172.21.15.22:6801/84729394":"2024-09-14T23:03:45.497872+0000","172.21.15.22:0/777163445":"2024-09-14T22:59:14.574157+0000","172.21.15.22:0/1891390217":"2024-09-14T23:03:45.497872+0000","172.21.15.22:0/4186392471":"2024-09-14T22:59:14.574157+0000","172.21.15.22:6801/3032931192":"2024-09-14T22:59:14.574157+0000","172.21.15.22:6800/3032931192":"2024-09-14T22:59:14.574157+0000","172.21.15.22:6800/84729394":"2024-09-14T23:03:45.497872+0000","172.21.15.22:0/898169442":"2024-09-14T22:59:14.574157+0000","172.21.15.22:6801/137268489":"2024-09-14T22:58:51.068764+0000","172.21.15.22:0/1254370233":"2024-09-14T22:58:51.068764+0000","172.21.15.22:0/1724895572":"2024-09-14T22:58:51.068764+0000","172.21.15.22:6800/137268489":"2024-09-14T22:58:51.068764+0000","172.21.15.22:0/1746885615":"2024-09-14T22:58:51.068764+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T23:04:12.911 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:12 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1558787846' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T23:04:12.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:12 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1558787846' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T23:04:12.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:12 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1558787846' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T23:04:13.661 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T23:04:13.661 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T23:04:13.662 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T23:04:13.662 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T23:04:13.663 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T23:04:13.663 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T23:04:13.664 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T23:04:13.664 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T23:04:13.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:13 smithi022 ceph-mon[23758]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:13.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:13 smithi022 ceph-mon[28440]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:14.070 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:14.097 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:14.123 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:14.123 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:14.126 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:14.127 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:14.127 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:14.152 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:14.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:13 smithi076 ceph-mon[24587]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:15.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:14 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:15.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:14 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:15.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:14 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:16.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:15 smithi022 ceph-mon[28440]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:16.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:15 smithi022 ceph-mon[23758]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:15 smithi076 ceph-mon[24587]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:17.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:17 smithi022 ceph-mon[23758]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:17.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:17 smithi022 ceph-mon[28440]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:17 smithi076 ceph-mon[24587]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:18.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:04:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:04:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:04:19.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:19 smithi022 ceph-mon[23758]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:19.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:19 smithi022 ceph-mon[28440]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:19 smithi076 ceph-mon[24587]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:20.817 INFO:teuthology.orchestra.run.smithi022.stdout:236223201292 2024-09-13T23:04:20.818 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T23:04:21.576 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:04:21 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:04:21] "GET /metrics HTTP/1.1" 200 191480 "" "Prometheus/2.33.4" 2024-09-13T23:04:22.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:21 smithi022 ceph-mon[23758]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:22.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:21 smithi022 ceph-mon[28440]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:22.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:21 smithi076 ceph-mon[24587]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:22.711 INFO:teuthology.orchestra.run.smithi022.stdout:55834574890 2024-09-13T23:04:22.712 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T23:04:23.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:23 smithi022 ceph-mon[23758]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:23.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:23 smithi022 ceph-mon[28440]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:24.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:23 smithi076 ceph-mon[24587]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:25.102 INFO:teuthology.orchestra.run.smithi022.stdout:201863462930 2024-09-13T23:04:25.102 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T23:04:26.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:25 smithi022 ceph-mon[23758]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:26.155 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:25 smithi022 ceph-mon[28440]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:25 smithi076 ceph-mon[24587]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:26.672 INFO:teuthology.orchestra.run.smithi022.stdout:34359738416 2024-09-13T23:04:26.673 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T23:04:27.933 INFO:teuthology.orchestra.run.smithi022.stdout:167503724567 2024-09-13T23:04:27.933 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T23:04:28.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:27 smithi022 ceph-mon[23758]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:28.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:27 smithi022 ceph-mon[28440]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:27 smithi076 ceph-mon[24587]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:28.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:04:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:04:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:04:29.636 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:29.636 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:29.654 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:29.654 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:29.932 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:29 smithi022 ceph-mon[23758]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:29.932 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:29 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:29.932 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:29 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:29.932 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:29 smithi022 ceph-mon[28440]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:29.932 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:29 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:29.932 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:29 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:30.126 INFO:teuthology.orchestra.run.smithi022.stdout:133143986203 2024-09-13T23:04:30.127 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T23:04:30.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:29 smithi076 ceph-mon[24587]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:30.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:29 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:30.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:29 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:31.108 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:30 smithi022 ceph-mon[23758]: Deploying daemon alertmanager.a on smithi022 2024-09-13T23:04:31.108 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:30 smithi022 ceph-mon[28440]: Deploying daemon alertmanager.a on smithi022 2024-09-13T23:04:31.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:30 smithi076 ceph-mon[24587]: Deploying daemon alertmanager.a on smithi022 2024-09-13T23:04:31.349 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:31.739 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:04:31 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:04:31] "GET /metrics HTTP/1.1" 200 191466 "" "Prometheus/2.33.4" 2024-09-13T23:04:32.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:31 smithi022 ceph-mon[23758]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:32.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:31 smithi022 ceph-mon[28440]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:32.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:31 smithi076 ceph-mon[24587]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:32.295 INFO:teuthology.orchestra.run.smithi022.stdout:77309411366 2024-09-13T23:04:32.295 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T23:04:33.781 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:34.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:33 smithi022 ceph-mon[23758]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:34.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:33 smithi022 ceph-mon[28440]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:33 smithi076 ceph-mon[24587]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:35.561 INFO:teuthology.orchestra.run.smithi022.stdout:111669149730 2024-09-13T23:04:35.561 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T23:04:36.106 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:35 smithi022 ceph-mon[23758]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:36.106 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:35 smithi022 ceph-mon[28440]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:35 smithi076 ceph-mon[24587]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:37.113 INFO:teuthology.orchestra.run.smithi022.stdout:201863462932 2024-09-13T23:04:37.292 INFO:teuthology.orchestra.run.smithi022.stdout:167503724569 2024-09-13T23:04:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:37 smithi022 ceph-mon[23758]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:37 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/712914944' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T23:04:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:37 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/4148293502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T23:04:38.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:37 smithi022 ceph-mon[28440]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:38.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:37 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/712914944' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T23:04:38.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:37 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/4148293502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T23:04:38.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:04:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:04:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:04:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:37 smithi076 ceph-mon[24587]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:37 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/712914944' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T23:04:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:37 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/4148293502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T23:04:38.365 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:39 smithi022 ceph-mon[23758]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:40.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:39 smithi022 ceph-mon[28440]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:39 smithi076 ceph-mon[24587]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:41.881 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:04:41 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:04:41] "GET /metrics HTTP/1.1" 200 191466 "" "Prometheus/2.33.4" 2024-09-13T23:04:42.151 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:41 smithi022 ceph-mon[28440]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:42.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:41 smithi022 ceph-mon[23758]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:41 smithi076 ceph-mon[24587]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:42.549 INFO:teuthology.orchestra.run.smithi022.stdout:34359738419 2024-09-13T23:04:43.122 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:42 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3915173312' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T23:04:43.123 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:42 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3915173312' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T23:04:43.130 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:43.147 INFO:teuthology.orchestra.run.smithi022.stdout:55834574894 2024-09-13T23:04:43.157 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724567 got 167503724569 for osd.5 2024-09-13T23:04:43.157 DEBUG:teuthology.parallel:result is None 2024-09-13T23:04:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:42 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3915173312' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T23:04:44.089 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:43 smithi022 ceph-mon[28440]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:44.089 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:43 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2150216578' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T23:04:44.089 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:43 smithi022 ceph-mon[23758]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:44.090 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:43 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2150216578' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T23:04:44.139 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462930 got 201863462932 for osd.6 2024-09-13T23:04:44.139 DEBUG:teuthology.parallel:result is None 2024-09-13T23:04:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:43 smithi076 ceph-mon[24587]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:43 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2150216578' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T23:04:45.114 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738419 for osd.0 2024-09-13T23:04:45.114 DEBUG:teuthology.parallel:result is None 2024-09-13T23:04:46.167 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:45 smithi022 ceph-mon[23758]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:46.167 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:45 smithi022 ceph-mon[28440]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:45 smithi076 ceph-mon[24587]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:46.849 INFO:teuthology.orchestra.run.smithi022.stdout:236223201297 2024-09-13T23:04:47.140 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:04:47.141 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:04:47.141 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:46 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1960691231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T23:04:47.141 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:04:47.141 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:04:47.141 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1960691231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T23:04:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:04:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:04:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:46 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1960691231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T23:04:47.626 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201292 got 236223201297 for osd.7 2024-09-13T23:04:47.626 DEBUG:teuthology.parallel:result is None 2024-09-13T23:04:48.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:47 smithi022 ceph-mon[23758]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:48.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:47 smithi022 ceph-mon[28440]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:48.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:47 smithi076 ceph-mon[24587]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:48.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:04:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:04:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:04:49.920 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574894 for osd.1 2024-09-13T23:04:49.921 DEBUG:teuthology.parallel:result is None 2024-09-13T23:04:50.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:49 smithi022 ceph-mon[23758]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:50.153 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:49 smithi022 ceph-mon[28440]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:49 smithi076 ceph-mon[24587]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:50.370 INFO:teuthology.orchestra.run.smithi022.stdout:133143986208 2024-09-13T23:04:50.583 INFO:teuthology.orchestra.run.smithi022.stdout:77309411370 2024-09-13T23:04:51.084 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:50 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2695232469' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T23:04:51.085 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:50 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2424041320' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T23:04:51.085 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:50 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2695232469' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T23:04:51.085 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:50 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2424041320' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T23:04:51.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:50 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2695232469' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T23:04:51.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:50 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2424041320' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T23:04:51.308 INFO:teuthology.orchestra.run.smithi022.stdout:111669149733 2024-09-13T23:04:51.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:04:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:04:51] "GET /metrics HTTP/1.1" 200 191475 "" "Prometheus/2.33.4" 2024-09-13T23:04:52.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:51 smithi022 ceph-mon[23758]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:52.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:51 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3562395536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T23:04:52.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:51 smithi022 ceph-mon[28440]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:52.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:51 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3562395536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T23:04:52.222 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411366 got 77309411370 for osd.2 2024-09-13T23:04:52.222 DEBUG:teuthology.parallel:result is None 2024-09-13T23:04:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:51 smithi076 ceph-mon[24587]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:51 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3562395536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T23:04:52.652 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986203 got 133143986208 for osd.4 2024-09-13T23:04:52.652 DEBUG:teuthology.parallel:result is None 2024-09-13T23:04:53.074 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149730 got 111669149733 for osd.3 2024-09-13T23:04:53.074 DEBUG:teuthology.parallel:result is None 2024-09-13T23:04:53.074 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T23:04:53.075 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T23:04:53.304 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 podman[46506]: 2024-09-13 23:04:53.220455609 +0000 UTC m=+0.967220257 volume create 40243fa1178efcda8021bcb38fdd30ecb8b36cb38d11bc5d70e9b5c10591687b 2024-09-13T23:04:53.304 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 podman[46506]: 2024-09-13 23:04:53.304212397 +0000 UTC m=+1.050977090 container create 64a47e17a5b1afd2c0495f486d4bced19080f8236cd9f9378fa40619c2699dda (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:04:53.720 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:53.905 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-mon[23758]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:53.905 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 podman[46506]: 2024-09-13 23:04:53.623333621 +0000 UTC m=+1.370098268 container init 64a47e17a5b1afd2c0495f486d4bced19080f8236cd9f9378fa40619c2699dda (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:04:53.905 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 podman[46506]: 2024-09-13 23:04:53.62637946 +0000 UTC m=+1.373144114 container start 64a47e17a5b1afd2c0495f486d4bced19080f8236cd9f9378fa40619c2699dda (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:04:53.905 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[46568]: level=info ts=2024-09-13T23:04:53.637Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-09-13T23:04:53.905 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[46568]: level=info ts=2024-09-13T23:04:53.637Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-09-13T23:04:53.906 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[46568]: level=info ts=2024-09-13T23:04:53.638Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.22 port=9094 2024-09-13T23:04:53.906 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[46568]: level=info ts=2024-09-13T23:04:53.639Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T23:04:53.906 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[46568]: level=info ts=2024-09-13T23:04:53.664Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T23:04:53.906 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[46568]: level=info ts=2024-09-13T23:04:53.665Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T23:04:53.906 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[46568]: level=info ts=2024-09-13T23:04:53.667Z caller=main.go:518 msg=Listening address=:9093 2024-09-13T23:04:53.906 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[46568]: level=info ts=2024-09-13T23:04:53.667Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-09-13T23:04:53.906 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 bash[46506]: 64a47e17a5b1afd2c0495f486d4bced19080f8236cd9f9378fa40619c2699dda 2024-09-13T23:04:53.906 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:53 smithi022 systemd[1]: Started Ceph alertmanager.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:04:54.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:54.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:54.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:54.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T23:04:54.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:54.169 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-mon[28440]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:54.169 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:54.169 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:54.169 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:54.169 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T23:04:54.169 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:53 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:54.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:53 smithi076 ceph-mon[24587]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:54.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:53 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:54.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:53 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:54.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:53 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:54.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:53 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T23:04:54.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:53 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:55.158 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:54 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T23:04:55.158 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:54 smithi022 ceph-mon[23758]: Deploying daemon grafana.a on smithi076 2024-09-13T23:04:55.158 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:54 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T23:04:55.158 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:54 smithi022 ceph-mon[28440]: Deploying daemon grafana.a on smithi076 2024-09-13T23:04:55.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:54 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T23:04:55.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:54 smithi076 ceph-mon[24587]: Deploying daemon grafana.a on smithi076 2024-09-13T23:04:55.363 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:04:55.365 INFO:teuthology.orchestra.run.smithi022.stderr:dumped all 2024-09-13T23:04:55.894 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:04:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[46568]: level=info ts=2024-09-13T23:04:55.640Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000123726s 2024-09-13T23:04:55.919 INFO:teuthology.orchestra.run.smithi022.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-09-13T23:04:54.322692+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48972,"kb_used_data":6156,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912244,"statfs":{"total":767960285184,"available":767910137856,"internally_reserved":0,"allocated":6303744,"data_stored":3510865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-13T23:03:46.397733+0000","last_change":"2024-09-13T23:03:29.311978+0000","last_active":"2024-09-13T23:03:46.397733+0000","last_peered":"2024-09-13T23:03:46.397733+0000","last_clean":"2024-09-13T23:03:46.397733+0000","last_became_active":"2024-09-13T23:03:29.311428+0000","last_became_peered":"2024-09-13T23:03:29.311428+0000","last_unstale":"2024-09-13T23:03:46.397733+0000","last_undegraded":"2024-09-13T23:03:46.397733+0000","last_fullsized":"2024-09-13T23:03:46.397733+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T23:01:29.222939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T23:01:29.222939+0000","last_clean_scrub_stamp":"2024-09-13T23:01:29.222939+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:01.747061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.026,"5min":1.026,"15min":1.026},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.53300000000000003}]},{"osd":1,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.498}]},{"osd":2,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.127,"5min":1.127,"15min":1.127},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.1659999999999999,"5min":1.1659999999999999,"15min":1.1659999999999999},"last":0.48299999999999998}]},{"osd":3,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.53900000000000003}]},{"osd":4,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.25,"5min":1.25,"15min":1.25},"last":0.373}]},{"osd":5,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.39300000000000002}]},{"osd":6,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.45000000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462936,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6160,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738992,"statfs":{"total":95995035648,"available":95988727808,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.49399999999999999}]},{"osd":1,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.432}]},{"osd":2,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.58199999999999996}]},{"osd":3,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.51400000000000001}]},{"osd":4,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.65600000000000003}]},{"osd":5,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.67600000000000005}]},{"osd":7,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.61099999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.38100000000000001,"15min":0.35799999999999998},"min":{"1min":0.27100000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68300000000000005,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.39000000000000001,"15min":0.35899999999999999},"min":{"1min":0.253,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.34300000000000003}]},{"osd":2,"last update":"Fri Sep 13 23:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.42499999999999999,"15min":0.41899999999999998},"min":{"1min":0.28799999999999998,"5min":0.255,"15min":0.255},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.43099999999999999}]},{"osd":3,"last update":"Fri Sep 13 23:04:00 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.46000000000000002,"15min":0.45400000000000001},"min":{"1min":0.32400000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.48499999999999999,"15min":0.47899999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.48299999999999998}]},{"osd":4,"last update":"Fri Sep 13 23:04:17 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.59699999999999998,"15min":0.59099999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.59999999999999998,"15min":0.59199999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.63300000000000001}]},{"osd":5,"last update":"Fri Sep 13 23:04:39 2024","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.65600000000000003,"15min":0.65000000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.66700000000000004,"15min":0.66300000000000003},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.59999999999999998}]},{"osd":6,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.55200000000000005}]},{"osd":7,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.68600000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 23:04:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.35499999999999998,"15min":0.33700000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.55800000000000005,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.378,"15min":0.35099999999999998},"min":{"1min":0.25700000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.63500000000000001,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.48799999999999999}]},{"osd":2,"last update":"Fri Sep 13 23:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41399999999999998,"15min":0.40000000000000002},"min":{"1min":0.28799999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.64400000000000002,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.432},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.41799999999999998,"15min":0.40799999999999997},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.623,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.44900000000000001}]},{"osd":3,"last update":"Fri Sep 13 23:03:57 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.435,"15min":0.439},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.60299999999999998,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.503,"5min":0.51900000000000002,"15min":0.52100000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.755,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.52200000000000002}]},{"osd":4,"last update":"Fri Sep 13 23:04:16 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.58799999999999997,"15min":0.57999999999999996},"min":{"1min":0.44400000000000001,"5min":0.441,"15min":0.441},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.622,"5min":0.57699999999999996,"15min":0.56899999999999995},"min":{"1min":0.48999999999999999,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.58999999999999997}]},{"osd":5,"last update":"Fri Sep 13 23:04:39 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.59399999999999997,"15min":0.58399999999999996},"min":{"1min":0.41399999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.60999999999999999,"15min":0.60499999999999998},"min":{"1min":0.45900000000000002,"5min":0.44,"15min":0.44},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.60499999999999998}]},{"osd":6,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.752}]},{"osd":7,"last update":"Fri Sep 13 23:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.77300000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6276,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":585728,"data_stored":240049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:04:34 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.41699999999999998,"15min":0.42099999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.374},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.41599999999999998,"15min":0.41499999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.54700000000000004,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.34999999999999998}]},{"osd":1,"last update":"Fri Sep 13 23:04:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.55300000000000005,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.433,"15min":0.437},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.70299999999999996,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.28499999999999998}]},{"osd":3,"last update":"Fri Sep 13 23:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.495,"5min":0.48699999999999999,"15min":0.48599999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.33900000000000002}]},{"osd":4,"last update":"Fri Sep 13 23:04:21 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.63600000000000001,"15min":0.64200000000000002},"min":{"1min":0.41999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.495},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.60899999999999999,"15min":0.60499999999999998},"min":{"1min":0.42899999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.64000000000000001}]},{"osd":5,"last update":"Fri Sep 13 23:04:39 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.53700000000000003}]},{"osd":6,"last update":"Fri Sep 13 23:04:04 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.70899999999999996}]},{"osd":7,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.50700000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.38100000000000001,"15min":0.372},"min":{"1min":0.26100000000000001,"5min":0.188,"15min":0.188},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.37},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32200000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.41599999999999998}]},{"osd":1,"last update":"Fri Sep 13 23:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.43099999999999999,"15min":0.42699999999999999},"min":{"1min":0.27100000000000002,"5min":0.224,"15min":0.224},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.41799999999999998,"15min":0.41199999999999998},"min":{"1min":0.28100000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.495}]},{"osd":2,"last update":"Fri Sep 13 23:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.45800000000000002,"15min":0.45600000000000002},"min":{"1min":0.33000000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.434},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.442,"15min":0.44},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.40300000000000002}]},{"osd":4,"last update":"Fri Sep 13 23:04:18 2024","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.62,"15min":0.61099999999999999},"min":{"1min":0.435,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.86299999999999999,"5min":1.04,"15min":1.04},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.65000000000000002,"15min":0.64400000000000002},"min":{"1min":0.48899999999999999,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.90700000000000003,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.745}]},{"osd":5,"last update":"Fri Sep 13 23:04:39 2024","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.61499999999999999,"15min":0.60799999999999998},"min":{"1min":0.42399999999999999,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.95099999999999996,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.62,"5min":0.60099999999999998,"15min":0.59799999999999998},"min":{"1min":0.433,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.78800000000000003,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.52800000000000002}]},{"osd":6,"last update":"Fri Sep 13 23:04:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.624}]},{"osd":7,"last update":"Fri Sep 13 23:04:29 2024","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.58399999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.55900000000000005,"15min":0.56799999999999995},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39100000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.39500000000000002}]},{"osd":1,"last update":"Fri Sep 13 23:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54500000000000004,"15min":0.54900000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.374,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.63300000000000001}]},{"osd":2,"last update":"Fri Sep 13 23:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55800000000000005,"15min":0.56200000000000006},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.443}]},{"osd":3,"last update":"Fri Sep 13 23:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.57199999999999995,"15min":0.57699999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.55500000000000005,"15min":0.55700000000000005},"min":{"1min":0.40600000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.51400000000000001}]},{"osd":5,"last update":"Fri Sep 13 23:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.433,"15min":0.42699999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.501,"5min":0.438,"15min":0.42799999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.55100000000000005}]},{"osd":6,"last update":"Fri Sep 13 23:04:05 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.66900000000000004}]},{"osd":7,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.61599999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.51500000000000001,"15min":0.51400000000000001},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.66700000000000004,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.436},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.49099999999999999,"15min":0.495},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.61699999999999999,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.35899999999999999}]},{"osd":1,"last update":"Fri Sep 13 23:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.48799999999999999,"15min":0.48199999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.37},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.501,"15min":0.50700000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.64600000000000002,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.47999999999999998}]},{"osd":2,"last update":"Fri Sep 13 23:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.495,"15min":0.495},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67400000000000004,"5min":0.755,"15min":0.755},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.67800000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.57399999999999995}]},{"osd":3,"last update":"Fri Sep 13 23:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.67500000000000004,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.38800000000000001,"5min":0.371,"15min":0.371},"max":{"1min":0.68300000000000005,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.64100000000000001}]},{"osd":4,"last update":"Fri Sep 13 23:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.39400000000000002,"15min":0.39200000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.42499999999999999,"15min":0.42099999999999999},"min":{"1min":0.28699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.69199999999999995,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.55700000000000005}]},{"osd":6,"last update":"Fri Sep 13 23:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.59999999999999998}]},{"osd":7,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.61299999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T23:04:55.921 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T23:04:56.134 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:56.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:55 smithi022 ceph-mon[28440]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:56.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:55 smithi022 ceph-mon[23758]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:56.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:55 smithi076 ceph-mon[24587]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:57.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:56 smithi022 ceph-mon[23758]: from='client.24523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T23:04:57.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:56 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:57.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:56 smithi022 ceph-mon[28440]: from='client.24523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T23:04:57.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:56 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:57.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:56 smithi076 ceph-mon[24587]: from='client.24523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T23:04:57.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:56 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:04:57.890 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:04:57.892 INFO:teuthology.orchestra.run.smithi022.stderr:dumped all 2024-09-13T23:04:58.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:57 smithi022 ceph-mon[28440]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:58.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:57 smithi022 ceph-mon[23758]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:57 smithi076 ceph-mon[24587]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:04:58.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:04:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:04:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:04:58.696 INFO:teuthology.orchestra.run.smithi022.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-09-13T23:04:56.323111+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48972,"kb_used_data":6156,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912244,"statfs":{"total":767960285184,"available":767910137856,"internally_reserved":0,"allocated":6303744,"data_stored":3510865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002558"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-13T23:03:46.397733+0000","last_change":"2024-09-13T23:03:29.311978+0000","last_active":"2024-09-13T23:03:46.397733+0000","last_peered":"2024-09-13T23:03:46.397733+0000","last_clean":"2024-09-13T23:03:46.397733+0000","last_became_active":"2024-09-13T23:03:29.311428+0000","last_became_peered":"2024-09-13T23:03:29.311428+0000","last_unstale":"2024-09-13T23:03:46.397733+0000","last_undegraded":"2024-09-13T23:03:46.397733+0000","last_fullsized":"2024-09-13T23:03:46.397733+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T23:01:29.222939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T23:01:29.222939+0000","last_clean_scrub_stamp":"2024-09-13T23:01:29.222939+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:01.747061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.026,"5min":1.026,"15min":1.026},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.53300000000000003}]},{"osd":1,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.498}]},{"osd":2,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.127,"5min":1.127,"15min":1.127},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.1659999999999999,"5min":1.1659999999999999,"15min":1.1659999999999999},"last":0.48299999999999998}]},{"osd":3,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.53900000000000003}]},{"osd":4,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.25,"5min":1.25,"15min":1.25},"last":0.373}]},{"osd":5,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.39300000000000002}]},{"osd":6,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.45000000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462936,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6160,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738992,"statfs":{"total":95995035648,"available":95988727808,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.49399999999999999}]},{"osd":1,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.432}]},{"osd":2,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.58199999999999996}]},{"osd":3,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.51400000000000001}]},{"osd":4,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.65600000000000003}]},{"osd":5,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.67600000000000005}]},{"osd":7,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.61099999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.38100000000000001,"15min":0.35799999999999998},"min":{"1min":0.27100000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68300000000000005,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.39000000000000001,"15min":0.35899999999999999},"min":{"1min":0.253,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.34300000000000003}]},{"osd":2,"last update":"Fri Sep 13 23:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.42499999999999999,"15min":0.41899999999999998},"min":{"1min":0.28799999999999998,"5min":0.255,"15min":0.255},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.43099999999999999}]},{"osd":3,"last update":"Fri Sep 13 23:04:00 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.46000000000000002,"15min":0.45400000000000001},"min":{"1min":0.32400000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.48499999999999999,"15min":0.47899999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.48299999999999998}]},{"osd":4,"last update":"Fri Sep 13 23:04:17 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.59699999999999998,"15min":0.59099999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.59999999999999998,"15min":0.59199999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.63300000000000001}]},{"osd":5,"last update":"Fri Sep 13 23:04:39 2024","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.65600000000000003,"15min":0.65000000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.66700000000000004,"15min":0.66300000000000003},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.59999999999999998}]},{"osd":6,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.55200000000000005}]},{"osd":7,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.68600000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 23:04:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.35499999999999998,"15min":0.33700000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.55800000000000005,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.378,"15min":0.35099999999999998},"min":{"1min":0.25700000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.63500000000000001,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.48799999999999999}]},{"osd":2,"last update":"Fri Sep 13 23:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41399999999999998,"15min":0.40000000000000002},"min":{"1min":0.28799999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.64400000000000002,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.432},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.41799999999999998,"15min":0.40799999999999997},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.623,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.44900000000000001}]},{"osd":3,"last update":"Fri Sep 13 23:03:57 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.435,"15min":0.439},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.60299999999999998,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.503,"5min":0.51900000000000002,"15min":0.52100000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.755,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.52200000000000002}]},{"osd":4,"last update":"Fri Sep 13 23:04:16 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.58799999999999997,"15min":0.57999999999999996},"min":{"1min":0.44400000000000001,"5min":0.441,"15min":0.441},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.622,"5min":0.57699999999999996,"15min":0.56899999999999995},"min":{"1min":0.48999999999999999,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.58999999999999997}]},{"osd":5,"last update":"Fri Sep 13 23:04:39 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.59399999999999997,"15min":0.58399999999999996},"min":{"1min":0.41399999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.60999999999999999,"15min":0.60499999999999998},"min":{"1min":0.45900000000000002,"5min":0.44,"15min":0.44},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.60499999999999998}]},{"osd":6,"last update":"Fri Sep 13 23:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.752}]},{"osd":7,"last update":"Fri Sep 13 23:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.77300000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6276,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":585728,"data_stored":240049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:04:34 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.41699999999999998,"15min":0.42099999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.374},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.41599999999999998,"15min":0.41499999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.54700000000000004,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.34999999999999998}]},{"osd":1,"last update":"Fri Sep 13 23:04:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.55300000000000005,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.433,"15min":0.437},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.70299999999999996,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.28499999999999998}]},{"osd":3,"last update":"Fri Sep 13 23:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.495,"5min":0.48699999999999999,"15min":0.48599999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.33900000000000002}]},{"osd":4,"last update":"Fri Sep 13 23:04:21 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.63600000000000001,"15min":0.64200000000000002},"min":{"1min":0.41999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.495},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.60899999999999999,"15min":0.60499999999999998},"min":{"1min":0.42899999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.64000000000000001}]},{"osd":5,"last update":"Fri Sep 13 23:04:39 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.53700000000000003}]},{"osd":6,"last update":"Fri Sep 13 23:04:04 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.70899999999999996}]},{"osd":7,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.50700000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.38100000000000001,"15min":0.372},"min":{"1min":0.26100000000000001,"5min":0.188,"15min":0.188},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.37},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32200000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.41599999999999998}]},{"osd":1,"last update":"Fri Sep 13 23:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.43099999999999999,"15min":0.42699999999999999},"min":{"1min":0.27100000000000002,"5min":0.224,"15min":0.224},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.41799999999999998,"15min":0.41199999999999998},"min":{"1min":0.28100000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.495}]},{"osd":2,"last update":"Fri Sep 13 23:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.45800000000000002,"15min":0.45600000000000002},"min":{"1min":0.33000000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.434},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.442,"15min":0.44},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.40300000000000002}]},{"osd":4,"last update":"Fri Sep 13 23:04:18 2024","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.62,"15min":0.61099999999999999},"min":{"1min":0.435,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.86299999999999999,"5min":1.04,"15min":1.04},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.65000000000000002,"15min":0.64400000000000002},"min":{"1min":0.48899999999999999,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.90700000000000003,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.745}]},{"osd":5,"last update":"Fri Sep 13 23:04:39 2024","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.61499999999999999,"15min":0.60799999999999998},"min":{"1min":0.42399999999999999,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.95099999999999996,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.62,"5min":0.60099999999999998,"15min":0.59799999999999998},"min":{"1min":0.433,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.78800000000000003,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.52800000000000002}]},{"osd":6,"last update":"Fri Sep 13 23:04:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.624}]},{"osd":7,"last update":"Fri Sep 13 23:04:29 2024","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.58399999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.55900000000000005,"15min":0.56799999999999995},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39100000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.39500000000000002}]},{"osd":1,"last update":"Fri Sep 13 23:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54500000000000004,"15min":0.54900000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.374,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.63300000000000001}]},{"osd":2,"last update":"Fri Sep 13 23:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55800000000000005,"15min":0.56200000000000006},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.443}]},{"osd":3,"last update":"Fri Sep 13 23:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.57199999999999995,"15min":0.57699999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.55500000000000005,"15min":0.55700000000000005},"min":{"1min":0.40600000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.51400000000000001}]},{"osd":5,"last update":"Fri Sep 13 23:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.433,"15min":0.42699999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.501,"5min":0.438,"15min":0.42799999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.55100000000000005}]},{"osd":6,"last update":"Fri Sep 13 23:04:05 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.66900000000000004}]},{"osd":7,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.61599999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 23:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.51500000000000001,"15min":0.51400000000000001},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.66700000000000004,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.436},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.49099999999999999,"15min":0.495},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.61699999999999999,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.35899999999999999}]},{"osd":1,"last update":"Fri Sep 13 23:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.48799999999999999,"15min":0.48199999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.37},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.501,"15min":0.50700000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.64600000000000002,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.47999999999999998}]},{"osd":2,"last update":"Fri Sep 13 23:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.495,"15min":0.495},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67400000000000004,"5min":0.755,"15min":0.755},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.67800000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.57399999999999995}]},{"osd":3,"last update":"Fri Sep 13 23:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.67500000000000004,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.38800000000000001,"5min":0.371,"15min":0.371},"max":{"1min":0.68300000000000005,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.64100000000000001}]},{"osd":4,"last update":"Fri Sep 13 23:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.39400000000000002,"15min":0.39200000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.42499999999999999,"15min":0.42099999999999999},"min":{"1min":0.28699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.69199999999999995,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.55700000000000005}]},{"osd":6,"last update":"Fri Sep 13 23:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.59999999999999998}]},{"osd":7,"last update":"Fri Sep 13 23:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.61299999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T23:04:58.697 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T23:04:58.697 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T23:04:58.697 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T23:04:58.698 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T23:04:58.920 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T23:04:59.042 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:58 smithi022 ceph-mon[23758]: from='client.24529 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T23:04:59.042 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:58 smithi022 ceph-mon[28440]: from='client.24529 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T23:04:59.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:58 smithi076 ceph-mon[24587]: from='client.24529 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T23:05:00.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:04:59 smithi022 ceph-mon[23758]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:00.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:04:59 smithi022 ceph-mon[28440]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:04:59 smithi076 ceph-mon[24587]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:00.653 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:05:00.653 INFO:teuthology.orchestra.run.smithi022.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T23:05:00.924 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:00 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1001698244' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T23:05:01.161 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:00 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1001698244' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T23:05:01.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:00 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1001698244' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T23:05:01.230 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T23:05:01.230 INFO:tasks.cephadm:Setup complete, yielding 2024-09-13T23:05:01.230 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T23:05:01.241 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi022.front.sepia.ceph.com 2024-09-13T23:05:01.241 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest false --force' 2024-09-13T23:05:01.815 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:05:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:05:01] "GET /metrics HTTP/1.1" 200 191473 "" "Prometheus/2.33.4" 2024-09-13T23:05:02.160 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:01 smithi022 ceph-mon[23758]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:02.160 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:01 smithi022 ceph-mon[28440]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:02.241 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:01 smithi076 ceph-mon[24587]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:03.722 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[46568]: level=info ts=2024-09-13T23:05:03.641Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.001674768s 2024-09-13T23:05:03.723 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T23:05:03.733 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi022.front.sepia.ceph.com 2024-09-13T23:05:03.733 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-09-13T23:05:03.775 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:03 smithi076 systemd[1]: Starting Ceph grafana.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:05:04.034 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:03 smithi076 podman[40676]: 2024-09-13 23:05:03.676453066 +0000 UTC m=+0.015139092 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-09-13T23:05:04.034 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:03 smithi076 podman[40676]: 2024-09-13 23:05:03.783285366 +0000 UTC m=+0.121988452 container create c818389ee4e7a2a5c692ed6bb3d721f1c5615dd5cec36a2613efe839a4eb5a03 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, io.buildah.version=1.24.2, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, release=236.1648460182, vcs-type=git, io.openshift.tags=base rhel8, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-09-13T23:05:04.034 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 podman[40676]: 2024-09-13 23:05:04.002810481 +0000 UTC m=+0.341496496 container init c818389ee4e7a2a5c692ed6bb3d721f1c5615dd5cec36a2613efe839a4eb5a03 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.component=ubi8-container, description=Ceph Grafana Container, build-date=2022-03-28T10:36:18.413762, io.openshift.tags=base rhel8, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, io.buildah.version=1.24.2, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , release=236.1648460182) 2024-09-13T23:05:04.034 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 podman[40676]: 2024-09-13 23:05:04.005545444 +0000 UTC m=+0.344231462 container start c818389ee4e7a2a5c692ed6bb3d721f1c5615dd5cec36a2613efe839a4eb5a03 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, build-date=2022-03-28T10:36:18.413762, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, release=236.1648460182, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, maintainer=Paul Cuzner , description=Ceph Grafana Container, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, name=ubi8, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T23:05:04.034 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-09-13T23:05:04.035 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-09-13T23:05:04.284 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-mon[24587]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:04.284 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' 2024-09-13T23:05:04.285 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-09-13T23:05:04.285 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-09-13T23:05:04.285 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T23:05:04.285 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T23:05:04.285 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T23:05:04.285 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T23:05:04.285 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-09-13T23:05:04.285 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-09-13T23:05:04.285 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-09-13T23:05:04.285 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-09-13T23:05:04.285 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="App mode production" logger=settings 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-09-13T23:05:04.286 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-09-13T23:05:04.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-09-13T23:05:04.288 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-09-13T23:05:04.289 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-09-13T23:05:04.289 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-09-13T23:05:04.289 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-09-13T23:05:04.289 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-09-13T23:05:04.289 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-13T23:05:04.289 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-09-13T23:05:04.289 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-09-13T23:05:04.289 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-09-13T23:05:04.289 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-09-13T23:05:04.289 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-09-13T23:05:04.290 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 bash[40676]: c818389ee4e7a2a5c692ed6bb3d721f1c5615dd5cec36a2613efe839a4eb5a03 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 systemd[1]: Started Ceph grafana.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-09-13T23:05:04.291 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-09-13T23:05:04.292 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-09-13T23:05:04.292 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-09-13T23:05:04.292 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-09-13T23:05:04.292 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-09-13T23:05:04.292 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-09-13T23:05:04.292 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-09-13T23:05:04.292 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-09-13T23:05:04.292 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-09-13T23:05:04.293 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-09-13T23:05:04.294 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-09-13T23:05:04.295 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-09-13T23:05:04.296 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-09-13T23:05:04.297 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-09-13T23:05:04.298 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-09-13T23:05:04.299 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-09-13T23:05:04.300 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-09-13T23:05:04.301 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-09-13T23:05:04.301 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-09-13T23:05:04.301 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-09-13T23:05:04.301 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-09-13T23:05:04.344 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:04 smithi022 ceph-mon[23758]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:04.344 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:04 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' 2024-09-13T23:05:04.344 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:04 smithi022 ceph-mon[28440]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:04.345 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:04 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' 2024-09-13T23:05:04.537 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-09-13T23:05:04.537 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-09-13T23:05:04.538 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-09-13T23:05:04.538 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-13T23:05:04.538 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-09-13T23:05:04.538 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-09-13T23:05:04.538 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-13T23:05:04.538 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-09-13T23:05:04.538 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-09-13T23:05:04.538 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-09-13T23:05:04.538 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-09-13T23:05:04.538 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-09-13T23:05:04.538 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-09-13T23:05:04.538 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-09-13T23:05:04.539 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-09-13T23:05:04.539 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-09-13T23:05:04.539 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-09-13T23:05:04.539 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-09-13T23:05:04.539 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-09-13T23:05:04.539 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-09-13T23:05:04.539 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-09-13T23:05:04.539 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-09-13T23:05:04.539 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-09-13T23:05:04.539 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-09-13T23:05:04.539 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-09-13T23:05:04.539 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-09-13T23:05:04.540 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-09-13T23:05:04.540 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-09-13T23:05:04.540 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-13T23:05:04.540 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-13T23:05:04.540 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-13T23:05:04.540 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-09-13T23:05:04.540 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-13T23:05:04.540 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-09-13T23:05:04.540 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-09-13T23:05:04.540 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-09-13T23:05:04.540 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-09-13T23:05:04.540 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-09-13T23:05:04.541 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-09-13T23:05:04.542 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-09-13T23:05:04.542 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-09-13T23:05:04.542 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-09-13T23:05:04.542 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-09-13T23:05:04.543 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-09-13T23:05:04.543 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-09-13T23:05:04.543 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-09-13T23:05:04.543 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-13T23:05:04.543 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-13T23:05:04.543 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-13T23:05:04.543 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-13T23:05:04.543 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-09-13T23:05:04.543 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-09-13T23:05:04.544 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-09-13T23:05:04.544 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-09-13T23:05:04.544 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-13T23:05:04.544 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-09-13T23:05:04.544 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-09-13T23:05:04.544 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-09-13T23:05:04.544 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-09-13T23:05:04.544 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-09-13T23:05:04.544 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-09-13T23:05:04.544 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-09-13T23:05:04.544 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-09-13T23:05:04.544 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-09-13T23:05:04.545 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-09-13T23:05:04.545 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-09-13T23:05:04.545 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-09-13T23:05:04.545 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-09-13T23:05:04.545 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-09-13T23:05:04.545 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-09-13T23:05:04.545 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-13T23:05:04.545 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-13T23:05:04.545 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-09-13T23:05:04.545 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-09-13T23:05:04.545 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-09-13T23:05:04.545 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-09-13T23:05:04.546 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-09-13T23:05:04.547 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-09-13T23:05:04.548 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-09-13T23:05:04.548 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-09-13T23:05:04.548 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-09-13T23:05:04.548 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-09-13T23:05:04.548 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-09-13T23:05:04.548 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-09-13T23:05:04.548 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-09-13T23:05:04.548 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-09-13T23:05:04.548 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-09-13T23:05:04.548 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-09-13T23:05:04.548 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-09-13T23:05:04.548 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-09-13T23:05:04.549 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-09-13T23:05:04.549 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-09-13T23:05:04.549 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-09-13T23:05:04.549 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-09-13T23:05:04.549 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-09-13T23:05:04.549 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-09-13T23:05:04.549 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-13T23:05:04.549 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-13T23:05:04.549 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-13T23:05:04.549 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-09-13T23:05:04.549 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-09-13T23:05:04.549 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-13T23:05:04.550 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-13T23:05:04.551 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-09-13T23:05:04.552 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-09-13T23:05:04.552 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-09-13T23:05:04.552 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-09-13T23:05:04.552 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-09-13T23:05:04.552 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-13T23:05:04.552 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-09-13T23:05:04.552 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-09-13T23:05:04.552 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-09-13T23:05:04.552 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-09-13T23:05:04.552 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-09-13T23:05:04.552 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-09-13T23:05:04.552 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-09-13T23:05:04.553 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-09-13T23:05:04.554 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=493.455103ms 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-09-13T23:05:04.555 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-09-13T23:05:05.037 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-09-13T23:05:05.037 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-09-13T23:05:05.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-09-13T23:05:05.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-09-13T23:05:05.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-13T23:05:05.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T23:05:05.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-09-13T23:05:05.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T23:05:05.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:05:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:05:04+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-09-13T23:05:05.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:05 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:05.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:05 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:05.319 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:05 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:05.319 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:05 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:05.319 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:05 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:05.319 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:05 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:05.469 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:05 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:05.469 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:05 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:05.469 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:05 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:06.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:06 smithi022 ceph-mon[28440]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:06.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:06 smithi022 ceph-mon[23758]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:06.433 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:06 smithi076 ceph-mon[24587]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:07.217 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:05:07.217 INFO:teuthology.orchestra.run.smithi022.stdout: "id": "2a72116f-929f-4d36-aa8a-d958ab7de51e", 2024-09-13T23:05:07.218 INFO:teuthology.orchestra.run.smithi022.stdout: "name": "r", 2024-09-13T23:05:07.218 INFO:teuthology.orchestra.run.smithi022.stdout: "current_period": "d154e944-60dc-4b01-aca4-8cdd30dc5b33", 2024-09-13T23:05:07.218 INFO:teuthology.orchestra.run.smithi022.stdout: "epoch": 1 2024-09-13T23:05:07.218 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:05:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:07 smithi076 ceph-mon[24587]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T23:05:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:07 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3330525948' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-13T23:05:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:07 smithi076 ceph-mon[24587]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:07 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:07.662 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:07 smithi022 ceph-mon[23758]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T23:05:07.662 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:07 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3330525948' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-13T23:05:07.662 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:07 smithi022 ceph-mon[23758]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:07.662 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:07 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:07.662 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:07 smithi022 ceph-mon[28440]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T23:05:07.662 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:07 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3330525948' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-13T23:05:07.662 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:07 smithi022 ceph-mon[28440]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:07.662 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:07 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:08.183 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:05:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:05:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:05:08.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:08 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3330525948' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-13T23:05:08.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:08 smithi022 ceph-mon[23758]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T23:05:08.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:08 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:08.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:08 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3330525948' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-13T23:05:08.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:08 smithi022 ceph-mon[28440]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T23:05:08.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:08 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:08 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3330525948' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-13T23:05:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:08 smithi076 ceph-mon[24587]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T23:05:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:08 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:08.565 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-09-13T23:05:09.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:09 smithi076 ceph-mon[24587]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T23:05:09.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:09 smithi076 ceph-mon[24587]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-13T23:05:09.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:09 smithi022 ceph-mon[23758]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T23:05:09.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:09 smithi022 ceph-mon[23758]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-13T23:05:09.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:09 smithi022 ceph-mon[28440]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T23:05:09.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:09 smithi022 ceph-mon[28440]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-13T23:05:10.308 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "id": "17bb3374-7e56-453d-bd1d-b44262b9441c", 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "name": "default", 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "api_name": "default", 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "is_master": "true", 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "endpoints": [], 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "hostnames": [], 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "hostnames_s3website": [], 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "master_zone": "", 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "zones": [], 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "placement_targets": [], 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "default_placement": "", 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "realm_id": "2a72116f-929f-4d36-aa8a-d958ab7de51e", 2024-09-13T23:05:10.309 INFO:teuthology.orchestra.run.smithi022.stdout: "sync_policy": { 2024-09-13T23:05:10.310 INFO:teuthology.orchestra.run.smithi022.stdout: "groups": [] 2024-09-13T23:05:10.310 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:05:10.310 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:05:10.897 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-09-13T23:05:11.871 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:05:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:05:11] "GET /metrics HTTP/1.1" 200 191473 "" "Prometheus/2.33.4" 2024-09-13T23:05:12.124 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:12 smithi022 ceph-mon[28440]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-13T23:05:12.124 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:12 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:12.124 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:12 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:12.125 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:12 smithi022 ceph-mon[23758]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-13T23:05:12.125 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:12 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:12.125 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:12 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:12 smithi076 ceph-mon[24587]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-13T23:05:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:12 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:12 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:13.226 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:05:13.226 INFO:teuthology.orchestra.run.smithi022.stdout: "id": "24dec3d7-0450-4412-b6db-7f1412654de4", 2024-09-13T23:05:13.226 INFO:teuthology.orchestra.run.smithi022.stdout: "name": "z", 2024-09-13T23:05:13.226 INFO:teuthology.orchestra.run.smithi022.stdout: "domain_root": "z.rgw.meta:root", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "control_pool": "z.rgw.control", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "gc_pool": "z.rgw.log:gc", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "lc_pool": "z.rgw.log:lc", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "log_pool": "z.rgw.log", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "roles_pool": "z.rgw.meta:roles", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-09-13T23:05:13.227 INFO:teuthology.orchestra.run.smithi022.stdout: "otp_pool": "z.rgw.otp", 2024-09-13T23:05:13.228 INFO:teuthology.orchestra.run.smithi022.stdout: "system_key": { 2024-09-13T23:05:13.228 INFO:teuthology.orchestra.run.smithi022.stdout: "access_key": "", 2024-09-13T23:05:13.228 INFO:teuthology.orchestra.run.smithi022.stdout: "secret_key": "" 2024-09-13T23:05:13.228 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:05:13.228 INFO:teuthology.orchestra.run.smithi022.stdout: "placement_pools": [ 2024-09-13T23:05:13.228 INFO:teuthology.orchestra.run.smithi022.stdout: { 2024-09-13T23:05:13.228 INFO:teuthology.orchestra.run.smithi022.stdout: "key": "default-placement", 2024-09-13T23:05:13.228 INFO:teuthology.orchestra.run.smithi022.stdout: "val": { 2024-09-13T23:05:13.228 INFO:teuthology.orchestra.run.smithi022.stdout: "index_pool": "z.rgw.buckets.index", 2024-09-13T23:05:13.228 INFO:teuthology.orchestra.run.smithi022.stdout: "storage_classes": { 2024-09-13T23:05:13.228 INFO:teuthology.orchestra.run.smithi022.stdout: "STANDARD": { 2024-09-13T23:05:13.228 INFO:teuthology.orchestra.run.smithi022.stdout: "data_pool": "z.rgw.buckets.data" 2024-09-13T23:05:13.229 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:05:13.229 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:05:13.229 INFO:teuthology.orchestra.run.smithi022.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-09-13T23:05:13.229 INFO:teuthology.orchestra.run.smithi022.stdout: "index_type": 0 2024-09-13T23:05:13.229 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:05:13.229 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:05:13.229 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-13T23:05:13.229 INFO:teuthology.orchestra.run.smithi022.stdout: "realm_id": "2a72116f-929f-4d36-aa8a-d958ab7de51e", 2024-09-13T23:05:13.229 INFO:teuthology.orchestra.run.smithi022.stdout: "notif_pool": "z.rgw.log:notif" 2024-09-13T23:05:13.229 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:05:13.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:13 smithi076 ceph-mon[24587]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T23:05:13.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:13 smithi076 ceph-mon[24587]: Reconfiguring daemon alertmanager.a on smithi022 2024-09-13T23:05:13.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:13 smithi022 ceph-mon[23758]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T23:05:13.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:13 smithi022 ceph-mon[23758]: Reconfiguring daemon alertmanager.a on smithi022 2024-09-13T23:05:13.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:13 smithi022 ceph-mon[28440]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T23:05:13.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:13 smithi022 ceph-mon[28440]: Reconfiguring daemon alertmanager.a on smithi022 2024-09-13T23:05:14.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:14 smithi022 ceph-mon[23758]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-13T23:05:14.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:14 smithi022 ceph-mon[28440]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-13T23:05:14.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:14 smithi076 ceph-mon[24587]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-13T23:05:14.489 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-09-13T23:05:16.221 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:16 smithi022 ceph-mon[23758]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-09-13T23:05:16.221 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:16 smithi022 ceph-mon[28440]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-09-13T23:05:16.221 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:16 smithi022 systemd[1]: Stopping Ceph alertmanager.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:05:16.221 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:16 smithi022 bash[48216]: Error: no container with name or ID "ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager.a" found: no such container 2024-09-13T23:05:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:16 smithi076 ceph-mon[24587]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-09-13T23:05:16.546 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:16 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[46568]: level=info ts=2024-09-13T23:05:16.220Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-09-13T23:05:16.546 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:16 smithi022 podman[48226]: 2024-09-13 23:05:16.240849448 +0000 UTC m=+0.134729184 container died 64a47e17a5b1afd2c0495f486d4bced19080f8236cd9f9378fa40619c2699dda (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:05:16.879 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:16 smithi022 podman[48226]: 2024-09-13 23:05:16.7536129 +0000 UTC m=+0.647492626 container cleanup 64a47e17a5b1afd2c0495f486d4bced19080f8236cd9f9378fa40619c2699dda (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:05:16.880 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:16 smithi022 bash[48226]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T23:05:16.880 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:16 smithi022 bash[48267]: Error: no container with name or ID "ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager.a" found: no such container 2024-09-13T23:05:17.525 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:17 smithi022 podman[48250]: 2024-09-13 23:05:17.30783438 +0000 UTC m=+1.061629772 container remove 64a47e17a5b1afd2c0495f486d4bced19080f8236cd9f9378fa40619c2699dda (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:05:17.526 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:17 smithi022 podman[48250]: 2024-09-13 23:05:17.408375359 +0000 UTC m=+1.162170717 volume remove 40243fa1178efcda8021bcb38fdd30ecb8b36cb38d11bc5d70e9b5c10591687b 2024-09-13T23:05:17.526 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:17 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T23:05:17.526 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:17 smithi022 systemd[1]: Stopped Ceph alertmanager.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:05:17.526 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:17 smithi022 systemd[1]: Starting Ceph alertmanager.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:05:18.112 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:18 smithi022 ceph-mon[23758]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 447 B/s rd, 1.2 KiB/s wr, 1 op/s 2024-09-13T23:05:18.113 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:18 smithi022 ceph-mon[28440]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 447 B/s rd, 1.2 KiB/s wr, 1 op/s 2024-09-13T23:05:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:18 smithi076 ceph-mon[24587]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 447 B/s rd, 1.2 KiB/s wr, 1 op/s 2024-09-13T23:05:18.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:05:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:05:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:05:19.072 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:18 smithi022 podman[48428]: 2024-09-13 23:05:18.722765969 +0000 UTC m=+0.024001269 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-09-13T23:05:19.072 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:18 smithi022 podman[48428]: 2024-09-13 23:05:18.824283899 +0000 UTC m=+0.125519193 volume create feabe4fe524a63762d0ecb1c6084ec4e2ccf78229de6803fae2aed1af3b4dbfb 2024-09-13T23:05:19.072 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:18 smithi022 podman[48428]: 2024-09-13 23:05:18.91630123 +0000 UTC m=+0.217536523 container create 863a54ec4b873e6d13ae655068f9ec3b93afb42d53f22a93493d99aece579368 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:05:19.333 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:19 smithi022 podman[48428]: 2024-09-13 23:05:19.210415706 +0000 UTC m=+0.511650993 container init 863a54ec4b873e6d13ae655068f9ec3b93afb42d53f22a93493d99aece579368 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:05:19.334 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:19 smithi022 podman[48428]: 2024-09-13 23:05:19.213293251 +0000 UTC m=+0.514528538 container start 863a54ec4b873e6d13ae655068f9ec3b93afb42d53f22a93493d99aece579368 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:05:19.334 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=info ts=2024-09-13T23:05:19.224Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-09-13T23:05:19.334 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=info ts=2024-09-13T23:05:19.224Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-09-13T23:05:19.334 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=info ts=2024-09-13T23:05:19.226Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.22 port=9094 2024-09-13T23:05:19.334 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=info ts=2024-09-13T23:05:19.227Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T23:05:19.334 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=info ts=2024-09-13T23:05:19.251Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T23:05:19.334 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=info ts=2024-09-13T23:05:19.251Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T23:05:19.334 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=info ts=2024-09-13T23:05:19.254Z caller=main.go:518 msg=Listening address=:9093 2024-09-13T23:05:19.334 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=info ts=2024-09-13T23:05:19.254Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-09-13T23:05:19.334 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:19 smithi022 bash[48428]: 863a54ec4b873e6d13ae655068f9ec3b93afb42d53f22a93493d99aece579368 2024-09-13T23:05:19.335 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:19 smithi022 systemd[1]: Started Ceph alertmanager.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:05:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:20 smithi076 ceph-mon[24587]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 706 B/s rd, 807 B/s wr, 1 op/s 2024-09-13T23:05:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:20 smithi076 ceph-mon[24587]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T23:05:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:20 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2603249236' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-13T23:05:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:20 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-13T23:05:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:20 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:20.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:20 smithi022 ceph-mon[28440]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 706 B/s rd, 807 B/s wr, 1 op/s 2024-09-13T23:05:20.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:20 smithi022 ceph-mon[28440]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T23:05:20.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:20 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2603249236' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-13T23:05:20.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:20 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-13T23:05:20.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:20 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:20.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:20 smithi022 ceph-mon[23758]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 706 B/s rd, 807 B/s wr, 1 op/s 2024-09-13T23:05:20.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:20 smithi022 ceph-mon[23758]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T23:05:20.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:20 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2603249236' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-13T23:05:20.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:20 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-13T23:05:20.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:20 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:21.426 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:21 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=info ts=2024-09-13T23:05:21.228Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000885072s 2024-09-13T23:05:21.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:21 smithi022 ceph-mon[23758]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:05:21.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:21 smithi022 ceph-mon[23758]: Reconfiguring daemon prometheus.a on smithi076 2024-09-13T23:05:21.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:21 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-13T23:05:21.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:21 smithi022 ceph-mon[23758]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T23:05:21.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:21 smithi022 ceph-mon[28440]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:05:21.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:21 smithi022 ceph-mon[28440]: Reconfiguring daemon prometheus.a on smithi076 2024-09-13T23:05:21.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:21 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-13T23:05:21.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:21 smithi022 ceph-mon[28440]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T23:05:21.534 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:21 smithi076 ceph-mon[24587]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:05:21.535 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:21 smithi076 ceph-mon[24587]: Reconfiguring daemon prometheus.a on smithi076 2024-09-13T23:05:21.535 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:21 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-13T23:05:21.535 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:21 smithi076 ceph-mon[24587]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T23:05:21.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:05:21 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:05:21] "GET /metrics HTTP/1.1" 200 192980 "" "Prometheus/2.33.4" 2024-09-13T23:05:22.373 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-mon[24587]: pgmap v55: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 1023 B/s wr, 2 op/s 2024-09-13T23:05:22.373 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-mon[24587]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T23:05:22.373 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2603249236' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-13T23:05:22.373 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-13T23:05:22.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:22 smithi022 ceph-mon[23758]: pgmap v55: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 1023 B/s wr, 2 op/s 2024-09-13T23:05:22.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:22 smithi022 ceph-mon[23758]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T23:05:22.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:22 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2603249236' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-13T23:05:22.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:22 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-13T23:05:22.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:22 smithi022 ceph-mon[28440]: pgmap v55: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 1023 B/s wr, 2 op/s 2024-09-13T23:05:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:22 smithi022 ceph-mon[28440]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T23:05:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:22 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2603249236' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-13T23:05:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:22 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-13T23:05:22.660 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 bash[41360]: Error: no container with name or ID "ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus.a" found: no such container 2024-09-13T23:05:22.954 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[39691]: ts=2024-09-13T23:05:22.660Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T23:05:22.954 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[39691]: ts=2024-09-13T23:05:22.660Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-09-13T23:05:22.954 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[39691]: ts=2024-09-13T23:05:22.660Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-09-13T23:05:22.954 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[39691]: ts=2024-09-13T23:05:22.660Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-09-13T23:05:22.954 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[39691]: ts=2024-09-13T23:05:22.660Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-09-13T23:05:22.954 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[39691]: ts=2024-09-13T23:05:22.660Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-09-13T23:05:22.955 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[39691]: ts=2024-09-13T23:05:22.661Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T23:05:22.955 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[39691]: ts=2024-09-13T23:05:22.661Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T23:05:22.955 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[39691]: ts=2024-09-13T23:05:22.661Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-09-13T23:05:22.955 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[39691]: ts=2024-09-13T23:05:22.661Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T23:05:22.955 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[39691]: ts=2024-09-13T23:05:22.661Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-09-13T23:05:22.955 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[39691]: ts=2024-09-13T23:05:22.661Z caller=main.go:1066 level=info msg="See you next time!" 2024-09-13T23:05:22.955 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:22 smithi076 podman[41372]: 2024-09-13 23:05:22.670877573 +0000 UTC m=+0.117159598 container died 2a87844e36ec2f04f80bb0ec9b773481eba5f00ed14e42f81f0268293f234355 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:05:23.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-13T23:05:23.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-mon[24587]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T23:05:23.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 podman[41372]: 2024-09-13 23:05:23.052129424 +0000 UTC m=+0.498411444 container cleanup 2a87844e36ec2f04f80bb0ec9b773481eba5f00ed14e42f81f0268293f234355 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:05:23.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 bash[41372]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T23:05:23.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 bash[41398]: Error: no container with name or ID "ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus.a" found: no such container 2024-09-13T23:05:23.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:23 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-13T23:05:23.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:23 smithi022 ceph-mon[23758]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T23:05:23.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:23 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-13T23:05:23.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:23 smithi022 ceph-mon[28440]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T23:05:23.615 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 podman[41383]: 2024-09-13 23:05:23.303730219 +0000 UTC m=+0.634748183 container remove 2a87844e36ec2f04f80bb0ec9b773481eba5f00ed14e42f81f0268293f234355 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:05:23.615 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T23:05:23.615 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 systemd[1]: Stopped Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:05:23.615 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 systemd[1]: Starting Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:05:23.616 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 podman[41461]: 2024-09-13 23:05:23.429035069 +0000 UTC m=+0.016937138 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-09-13T23:05:23.616 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 podman[41461]: 2024-09-13 23:05:23.529680094 +0000 UTC m=+0.117582193 container create 8e9cd8b611e38b6d9905423b92cebc0c50cb59cb894d7a8553eb070a54a73fab (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:05:24.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 podman[41461]: 2024-09-13 23:05:23.732199511 +0000 UTC m=+0.320101574 container init 8e9cd8b611e38b6d9905423b92cebc0c50cb59cb894d7a8553eb070a54a73fab (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:05:24.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 podman[41461]: 2024-09-13 23:05:23.734903707 +0000 UTC m=+0.322805770 container start 8e9cd8b611e38b6d9905423b92cebc0c50cb59cb894d7a8553eb070a54a73fab (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:05:24.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.757Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-13T23:05:24.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.757Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-09-13T23:05:24.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.757Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-09-13T23:05:24.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.757Z caller=main.go:518 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi076 (none))" 2024-09-13T23:05:24.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.757Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T23:05:24.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.757Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T23:05:24.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.759Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T23:05:24.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.759Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-09-13T23:05:24.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.761Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-09-13T23:05:24.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.762Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T23:05:24.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.762Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.634µs 2024-09-13T23:05:24.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.762Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T23:05:24.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 bash[41461]: 8e9cd8b611e38b6d9905423b92cebc0c50cb59cb894d7a8553eb070a54a73fab 2024-09-13T23:05:24.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 systemd[1]: Started Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:05:24.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.951Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-13T23:05:24.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.951Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-13T23:05:24.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.951Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=25.273µs wal_replay_duration=189.095048ms total_replay_duration=189.133187ms 2024-09-13T23:05:24.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.954Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T23:05:24.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.955Z caller=main.go:947 level=info msg="TSDB started" 2024-09-13T23:05:24.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.955Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T23:05:24.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.969Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.355668ms db_storage=866ns remote_storage=1.849µs web_handler=385ns query_engine=751ns scrape=483.943µs scrape_sd=36.198µs notify=27.914µs notify_sd=11.432µs rules=13.486839ms 2024-09-13T23:05:24.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:05:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:05:23.969Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-09-13T23:05:24.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: pgmap v58: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:24.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T23:05:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-13T23:05:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T23:05:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.22:9093"}]: dispatch 2024-09-13T23:05:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:05:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.76:9095"}]: dispatch 2024-09-13T23:05:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:05:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.76:3000"}]: dispatch 2024-09-13T23:05:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:24.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:24.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: pgmap v58: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:24.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T23:05:24.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-13T23:05:24.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:24.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T23:05:24.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.22:9093"}]: dispatch 2024-09-13T23:05:24.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:24.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:05:24.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.76:9095"}]: dispatch 2024-09-13T23:05:24.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:24.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:05:24.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.76:3000"}]: dispatch 2024-09-13T23:05:24.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:24.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:24.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:24 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:24.298 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: pgmap v58: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:24.298 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T23:05:24.298 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-13T23:05:24.298 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:24.298 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T23:05:24.298 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.22:9093"}]: dispatch 2024-09-13T23:05:24.298 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:24.298 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:05:24.298 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.76:9095"}]: dispatch 2024-09-13T23:05:24.298 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:24.299 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:05:24.299 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.76:3000"}]: dispatch 2024-09-13T23:05:24.299 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:24.299 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:24.299 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:24 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T23:05:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.22:9093"}]: dispatch 2024-09-13T23:05:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:05:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.76:9095"}]: dispatch 2024-09-13T23:05:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:05:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.76:3000"}]: dispatch 2024-09-13T23:05:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-13T23:05:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[28440]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T23:05:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-13T23:05:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-13T23:05:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[28440]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T23:05:25.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-13T23:05:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T23:05:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.22:9093"}]: dispatch 2024-09-13T23:05:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:05:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.76:9095"}]: dispatch 2024-09-13T23:05:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:05:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.76:3000"}]: dispatch 2024-09-13T23:05:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-13T23:05:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[23758]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T23:05:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-13T23:05:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-13T23:05:25.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[23758]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T23:05:25.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:25 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-13T23:05:25.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:25 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T23:05:25.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:25 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.22:9093"}]: dispatch 2024-09-13T23:05:25.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:25 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:05:25.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:25 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.76:9095"}]: dispatch 2024-09-13T23:05:25.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:25 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:05:25.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:25 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.76:3000"}]: dispatch 2024-09-13T23:05:25.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:25 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-13T23:05:25.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:25 smithi076 ceph-mon[24587]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T23:05:25.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:25 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-13T23:05:25.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:25 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-13T23:05:25.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:25 smithi076 ceph-mon[24587]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T23:05:25.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:25 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-13T23:05:26.186 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:05:26.186 INFO:teuthology.orchestra.run.smithi022.stdout: "id": "c3eadd0d-cd68-43fc-aebd-ccd7de558278", 2024-09-13T23:05:26.186 INFO:teuthology.orchestra.run.smithi022.stdout: "epoch": 1, 2024-09-13T23:05:26.186 INFO:teuthology.orchestra.run.smithi022.stdout: "predecessor_uuid": "d154e944-60dc-4b01-aca4-8cdd30dc5b33", 2024-09-13T23:05:26.186 INFO:teuthology.orchestra.run.smithi022.stdout: "sync_status": [], 2024-09-13T23:05:26.186 INFO:teuthology.orchestra.run.smithi022.stdout: "period_map": { 2024-09-13T23:05:26.186 INFO:teuthology.orchestra.run.smithi022.stdout: "id": "c3eadd0d-cd68-43fc-aebd-ccd7de558278", 2024-09-13T23:05:26.186 INFO:teuthology.orchestra.run.smithi022.stdout: "zonegroups": [ 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: { 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: "id": "17bb3374-7e56-453d-bd1d-b44262b9441c", 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: "name": "default", 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: "api_name": "default", 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: "is_master": "true", 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: "endpoints": [], 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: "hostnames": [], 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: "hostnames_s3website": [], 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: "master_zone": "24dec3d7-0450-4412-b6db-7f1412654de4", 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: "zones": [ 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: { 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: "id": "24dec3d7-0450-4412-b6db-7f1412654de4", 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: "name": "z", 2024-09-13T23:05:26.187 INFO:teuthology.orchestra.run.smithi022.stdout: "endpoints": [], 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: "log_meta": "false", 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: "log_data": "false", 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: "bucket_index_max_shards": 11, 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: "read_only": "false", 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: "tier_type": "", 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: "sync_from_all": "true", 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: "sync_from": [], 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: "redirect_zone": "" 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: "placement_targets": [ 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: { 2024-09-13T23:05:26.188 INFO:teuthology.orchestra.run.smithi022.stdout: "name": "default-placement", 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: "tags": [], 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: "storage_classes": [ 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: "STANDARD" 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: ] 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: "default_placement": "default-placement", 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: "realm_id": "2a72116f-929f-4d36-aa8a-d958ab7de51e", 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: "sync_policy": { 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: "groups": [] 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:05:26.189 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: "short_zone_ids": [ 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: { 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: "key": "24dec3d7-0450-4412-b6db-7f1412654de4", 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: "val": 3910210161 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: ] 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: "master_zonegroup": "17bb3374-7e56-453d-bd1d-b44262b9441c", 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: "master_zone": "24dec3d7-0450-4412-b6db-7f1412654de4", 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: "period_config": { 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: "bucket_quota": { 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: "enabled": false, 2024-09-13T23:05:26.190 INFO:teuthology.orchestra.run.smithi022.stdout: "check_on_raw": false, 2024-09-13T23:05:26.191 INFO:teuthology.orchestra.run.smithi022.stdout: "max_size": -1, 2024-09-13T23:05:26.191 INFO:teuthology.orchestra.run.smithi022.stdout: "max_size_kb": 0, 2024-09-13T23:05:26.191 INFO:teuthology.orchestra.run.smithi022.stdout: "max_objects": -1 2024-09-13T23:05:26.191 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:05:26.191 INFO:teuthology.orchestra.run.smithi022.stdout: "user_quota": { 2024-09-13T23:05:26.191 INFO:teuthology.orchestra.run.smithi022.stdout: "enabled": false, 2024-09-13T23:05:26.191 INFO:teuthology.orchestra.run.smithi022.stdout: "check_on_raw": false, 2024-09-13T23:05:26.191 INFO:teuthology.orchestra.run.smithi022.stdout: "max_size": -1, 2024-09-13T23:05:26.191 INFO:teuthology.orchestra.run.smithi022.stdout: "max_size_kb": 0, 2024-09-13T23:05:26.191 INFO:teuthology.orchestra.run.smithi022.stdout: "max_objects": -1 2024-09-13T23:05:26.191 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:05:26.191 INFO:teuthology.orchestra.run.smithi022.stdout: "user_ratelimit": { 2024-09-13T23:05:26.192 INFO:teuthology.orchestra.run.smithi022.stdout: "max_read_ops": 0, 2024-09-13T23:05:26.192 INFO:teuthology.orchestra.run.smithi022.stdout: "max_write_ops": 0, 2024-09-13T23:05:26.192 INFO:teuthology.orchestra.run.smithi022.stdout: "max_read_bytes": 0, 2024-09-13T23:05:26.192 INFO:teuthology.orchestra.run.smithi022.stdout: "max_write_bytes": 0, 2024-09-13T23:05:26.192 INFO:teuthology.orchestra.run.smithi022.stdout: "enabled": false 2024-09-13T23:05:26.192 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:05:26.192 INFO:teuthology.orchestra.run.smithi022.stdout: "bucket_ratelimit": { 2024-09-13T23:05:26.192 INFO:teuthology.orchestra.run.smithi022.stdout: "max_read_ops": 0, 2024-09-13T23:05:26.192 INFO:teuthology.orchestra.run.smithi022.stdout: "max_write_ops": 0, 2024-09-13T23:05:26.192 INFO:teuthology.orchestra.run.smithi022.stdout: "max_read_bytes": 0, 2024-09-13T23:05:26.192 INFO:teuthology.orchestra.run.smithi022.stdout: "max_write_bytes": 0, 2024-09-13T23:05:26.192 INFO:teuthology.orchestra.run.smithi022.stdout: "enabled": false 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout: "anonymous_ratelimit": { 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout: "max_read_ops": 0, 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout: "max_write_ops": 0, 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout: "max_read_bytes": 0, 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout: "max_write_bytes": 0, 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout: "enabled": false 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout: "realm_id": "2a72116f-929f-4d36-aa8a-d958ab7de51e", 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout: "realm_name": "r", 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout: "realm_epoch": 2 2024-09-13T23:05:26.193 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:05:26.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:26 smithi022 ceph-mon[28440]: pgmap v61: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-09-13T23:05:26.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:26 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-13T23:05:26.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:26 smithi022 ceph-mon[28440]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T23:05:26.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:26 smithi022 ceph-mon[23758]: pgmap v61: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-09-13T23:05:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:26 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-13T23:05:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:26 smithi022 ceph-mon[23758]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T23:05:26.436 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:26 smithi076 ceph-mon[24587]: pgmap v61: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-09-13T23:05:26.436 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:26 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2331876211' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-13T23:05:26.436 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:26 smithi076 ceph-mon[24587]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T23:05:27.213 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-09-13T23:05:27.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:27 smithi076 ceph-mon[24587]: pgmap v64: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-09-13T23:05:27.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:27 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:27.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:27 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:27.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:27 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:27.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:27 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:27.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:27 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:27.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:27 smithi022 ceph-mon[23758]: pgmap v64: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-09-13T23:05:27.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:27 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:27.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:27 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:27.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:27 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:27.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:27 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:27.827 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:27 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:27.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:27 smithi022 ceph-mon[28440]: pgmap v64: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-09-13T23:05:27.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:27 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:27.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:27 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:27.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:27 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:27.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:27 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:27.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:27 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:29.395 INFO:teuthology.orchestra.run.smithi022.stdout:Scheduled rgw.foo update... 2024-09-13T23:05:29.395 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:05:29 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=info ts=2024-09-13T23:05:29.229Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.00229754s 2024-09-13T23:05:29.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:29 smithi022 ceph-mon[23758]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-13T23:05:29.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:29 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:29.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:29 smithi022 ceph-mon[28440]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-13T23:05:29.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:29 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:29.786 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:29 smithi076 ceph-mon[24587]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-13T23:05:29.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:29 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:30.055 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph osd pool create foo' 2024-09-13T23:05:30.627 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:30 smithi022 ceph-mon[23758]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:05:30.627 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:30 smithi022 ceph-mon[23758]: Saving service rgw.foo spec with placement count:2 2024-09-13T23:05:30.628 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:30 smithi022 ceph-mon[28440]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:05:30.628 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:30 smithi022 ceph-mon[28440]: Saving service rgw.foo spec with placement count:2 2024-09-13T23:05:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:30 smithi076 ceph-mon[24587]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:05:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:30 smithi076 ceph-mon[24587]: Saving service rgw.foo spec with placement count:2 2024-09-13T23:05:31.737 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:05:31] "GET /metrics HTTP/1.1" 200 196831 "" "Prometheus/2.33.4" 2024-09-13T23:05:31.737 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[23758]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-13T23:05:31.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:31.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:31.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:31.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:31.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[23758]: Saving service rgw.foo spec with placement count:2 2024-09-13T23:05:31.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:31.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi022.snmrkm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:05:31.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi022.snmrkm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T23:05:31.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:31.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:31.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[23758]: Deploying daemon rgw.foo.smithi022.snmrkm on smithi022 2024-09-13T23:05:31.739 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[28440]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-13T23:05:31.739 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:31.739 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:31.739 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:31.739 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:31.739 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[28440]: Saving service rgw.foo spec with placement count:2 2024-09-13T23:05:31.739 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:31.739 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi022.snmrkm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:05:31.739 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi022.snmrkm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T23:05:31.739 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:31.739 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:31.739 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:31 smithi022 ceph-mon[28440]: Deploying daemon rgw.foo.smithi022.snmrkm on smithi022 2024-09-13T23:05:32.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:31 smithi076 ceph-mon[24587]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-13T23:05:32.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:31 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:32.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:31 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:32.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:31 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:32.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:31 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:32.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:31 smithi076 ceph-mon[24587]: Saving service rgw.foo spec with placement count:2 2024-09-13T23:05:32.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:31 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:32.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:31 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi022.snmrkm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:05:32.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:31 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi022.snmrkm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T23:05:32.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:31 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:32.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:31 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:32.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:31 smithi076 ceph-mon[24587]: Deploying daemon rgw.foo.smithi022.snmrkm on smithi022 2024-09-13T23:05:32.798 INFO:teuthology.orchestra.run.smithi022.stderr:pool 'foo' created 2024-09-13T23:05:32.963 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:32 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1911858269' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-13T23:05:32.963 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:32 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-13T23:05:32.963 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:32 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1911858269' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-13T23:05:32.963 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:32 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-13T23:05:33.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:32 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1911858269' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-13T23:05:33.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:32 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-13T23:05:33.895 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:33 smithi022 ceph-mon[23758]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-09-13T23:05:33.895 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:33 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-13T23:05:33.895 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:33 smithi022 ceph-mon[23758]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T23:05:33.895 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:33 smithi022 ceph-mon[28440]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-09-13T23:05:33.895 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:33 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-13T23:05:33.895 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:33 smithi022 ceph-mon[28440]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T23:05:33.919 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'rbd pool init foo' 2024-09-13T23:05:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:33 smithi076 ceph-mon[24587]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-09-13T23:05:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:33 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-13T23:05:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:33 smithi076 ceph-mon[24587]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T23:05:35.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:34 smithi076 ceph-mon[24587]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T23:05:35.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:34 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:35.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:34 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi076.rnamrn", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:05:35.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:34 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi076.rnamrn", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T23:05:35.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:34 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:35.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:34 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:35.103 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:34 smithi022 ceph-mon[28440]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T23:05:35.103 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:34 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:35.103 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:34 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi076.rnamrn", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:05:35.103 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:34 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi076.rnamrn", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T23:05:35.103 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:34 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:35.104 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:34 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:35.104 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:34 smithi022 ceph-mon[23758]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T23:05:35.104 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:34 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:35.104 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:34 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi076.rnamrn", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:05:35.104 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:34 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi076.rnamrn", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T23:05:35.104 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:34 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:35.104 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:34 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:35.824 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:35 smithi076 ceph-mon[24587]: pgmap v70: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-09-13T23:05:35.825 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:35 smithi076 ceph-mon[24587]: Deploying daemon rgw.foo.smithi076.rnamrn on smithi076 2024-09-13T23:05:35.825 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:35 smithi076 ceph-mon[24587]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T23:05:36.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:35 smithi022 ceph-mon[23758]: pgmap v70: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-09-13T23:05:36.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:35 smithi022 ceph-mon[23758]: Deploying daemon rgw.foo.smithi076.rnamrn on smithi076 2024-09-13T23:05:36.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:35 smithi022 ceph-mon[23758]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T23:05:36.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:35 smithi022 ceph-mon[28440]: pgmap v70: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-09-13T23:05:36.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:35 smithi022 ceph-mon[28440]: Deploying daemon rgw.foo.smithi076.rnamrn on smithi076 2024-09-13T23:05:36.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:35 smithi022 ceph-mon[28440]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T23:05:37.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:36 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2555628334' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-13T23:05:37.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:36 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-13T23:05:37.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:36 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2555628334' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-13T23:05:37.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:36 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-13T23:05:37.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:36 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2555628334' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-13T23:05:37.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:36 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-13T23:05:38.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:05:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:05:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:05:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:37 smithi076 ceph-mon[24587]: pgmap v72: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:37 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-13T23:05:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:37 smithi076 ceph-mon[24587]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T23:05:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:37 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:37 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:37 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:38.077 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:37 smithi022 ceph-mon[23758]: pgmap v72: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:38.077 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:37 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-13T23:05:38.078 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:37 smithi022 ceph-mon[23758]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T23:05:38.078 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:37 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:38.078 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:37 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:38.078 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:37 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:38.078 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:37 smithi022 ceph-mon[28440]: pgmap v72: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:05:38.078 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:37 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-13T23:05:38.078 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:37 smithi022 ceph-mon[28440]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T23:05:38.078 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:37 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:38.078 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:37 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:38.078 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:37 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:39.109 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:38 smithi076 ceph-mon[24587]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T23:05:39.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:38 smithi022 ceph-mon[28440]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T23:05:39.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:38 smithi022 ceph-mon[23758]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T23:05:39.609 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch apply iscsi foo u p' 2024-09-13T23:05:39.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:39 smithi022 ceph-mon[23758]: pgmap v75: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 7.3 KiB/s wr, 246 op/s 2024-09-13T23:05:39.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:39 smithi022 ceph-mon[23758]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T23:05:39.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:39 smithi022 ceph-mon[28440]: pgmap v75: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 7.3 KiB/s wr, 246 op/s 2024-09-13T23:05:39.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:39 smithi022 ceph-mon[28440]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T23:05:40.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:39 smithi076 ceph-mon[24587]: pgmap v75: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 7.3 KiB/s wr, 246 op/s 2024-09-13T23:05:40.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:39 smithi076 ceph-mon[24587]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T23:05:41.669 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:05:41 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:05:41] "GET /metrics HTTP/1.1" 200 196831 "" "Prometheus/2.33.4" 2024-09-13T23:05:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:41 smithi076 ceph-mon[24587]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 6.0 KiB/s wr, 202 op/s 2024-09-13T23:05:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:41 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:41 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:42.321 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:41 smithi022 ceph-mon[23758]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 6.0 KiB/s wr, 202 op/s 2024-09-13T23:05:42.321 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:41 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:42.322 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:41 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:42.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:41 smithi022 ceph-mon[28440]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 6.0 KiB/s wr, 202 op/s 2024-09-13T23:05:42.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:41 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:42.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:41 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:42.651 INFO:teuthology.orchestra.run.smithi022.stdout:Scheduled iscsi.foo update... 2024-09-13T23:05:43.344 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 180' 2024-09-13T23:05:43.901 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:43 smithi022 ceph-mon[23758]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s rd, 5.7 KiB/s wr, 265 op/s 2024-09-13T23:05:43.901 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:43 smithi022 ceph-mon[23758]: from='client.24664 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:05:43.901 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:43 smithi022 ceph-mon[23758]: Saving service iscsi.foo spec with placement count:1 2024-09-13T23:05:43.901 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:43 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:43.901 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:43 smithi022 ceph-mon[28440]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s rd, 5.7 KiB/s wr, 265 op/s 2024-09-13T23:05:43.901 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:43 smithi022 ceph-mon[28440]: from='client.24664 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:05:43.902 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:43 smithi022 ceph-mon[28440]: Saving service iscsi.foo spec with placement count:1 2024-09-13T23:05:43.902 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:43 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:44.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:43 smithi076 ceph-mon[24587]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s rd, 5.7 KiB/s wr, 265 op/s 2024-09-13T23:05:44.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:43 smithi076 ceph-mon[24587]: from='client.24664 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:05:44.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:43 smithi076 ceph-mon[24587]: Saving service iscsi.foo spec with placement count:1 2024-09-13T23:05:44.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:43 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.156 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.156 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[28440]: Checking dashboard <-> RGW credentials 2024-09-13T23:05:45.156 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.156 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.156 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.156 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[28440]: Checking pool "foo" exists for service iscsi.foo 2024-09-13T23:05:45.156 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:05:45.156 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-13T23:05:45.156 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:45.157 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[28440]: Deploying daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:05:45.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[23758]: Checking dashboard <-> RGW credentials 2024-09-13T23:05:45.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[23758]: Checking pool "foo" exists for service iscsi.foo 2024-09-13T23:05:45.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:05:45.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-13T23:05:45.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:45.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:44 smithi022 ceph-mon[23758]: Deploying daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:05:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:44 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:44 smithi076 ceph-mon[24587]: Checking dashboard <-> RGW credentials 2024-09-13T23:05:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:44 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:44 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:44 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:44 smithi076 ceph-mon[24587]: Checking pool "foo" exists for service iscsi.foo 2024-09-13T23:05:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:44 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:05:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:44 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-13T23:05:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:44 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:45.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:44 smithi076 ceph-mon[24587]: Deploying daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:05:46.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:46 smithi022 ceph-mon[23758]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 4.8 KiB/s wr, 282 op/s 2024-09-13T23:05:46.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:46 smithi022 ceph-mon[28440]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 4.8 KiB/s wr, 282 op/s 2024-09-13T23:05:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:46 smithi076 ceph-mon[24587]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 4.8 KiB/s wr, 282 op/s 2024-09-13T23:05:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: mgrmap e21: y(active, since 2m), standbys: x 2024-09-13T23:05:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-09-13T23:05:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-13T23:05:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-09-13T23:05:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-13T23:05:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:05:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:05:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-09-13T23:05:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-13T23:05:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-13T23:05:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-13T23:05:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[28440]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T23:05:47.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: mgrmap e21: y(active, since 2m), standbys: x 2024-09-13T23:05:47.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-09-13T23:05:47.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-13T23:05:47.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-09-13T23:05:47.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-13T23:05:47.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:05:47.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:05:47.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:47.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-09-13T23:05:47.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-13T23:05:47.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-13T23:05:47.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-13T23:05:47.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:47 smithi022 ceph-mon[23758]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T23:05:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: mgrmap e21: y(active, since 2m), standbys: x 2024-09-13T23:05:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-09-13T23:05:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-13T23:05:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-09-13T23:05:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-13T23:05:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:05:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:05:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-09-13T23:05:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-13T23:05:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-13T23:05:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-13T23:05:47.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-mon[24587]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T23:05:48.167 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:05:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:05:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:05:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:48 smithi022 ceph-mon[23758]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 360 B/s wr, 117 op/s 2024-09-13T23:05:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:48 smithi022 ceph-mon[23758]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T23:05:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:48 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:48.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:48 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:48.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:48 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:48 smithi022 ceph-mon[28440]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 360 B/s wr, 117 op/s 2024-09-13T23:05:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:48 smithi022 ceph-mon[28440]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T23:05:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:48 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:48 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:48 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:48 smithi076 ceph-mon[24587]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 360 B/s wr, 117 op/s 2024-09-13T23:05:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:48 smithi076 ceph-mon[24587]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T23:05:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:48 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:48 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:48 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:49.173 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:49 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3986285855' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:05:49.173 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:49 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/381370935' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1481920223"}]: dispatch 2024-09-13T23:05:49.173 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:49 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1481920223"}]: dispatch 2024-09-13T23:05:49.173 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:49 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3986285855' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:05:49.173 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:49 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/381370935' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1481920223"}]: dispatch 2024-09-13T23:05:49.174 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:49 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1481920223"}]: dispatch 2024-09-13T23:05:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:49 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3986285855' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:05:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:49 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/381370935' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1481920223"}]: dispatch 2024-09-13T23:05:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:49 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1481920223"}]: dispatch 2024-09-13T23:05:50.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:50 smithi022 ceph-mon[23758]: pgmap v83: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 639 B/s wr, 130 op/s 2024-09-13T23:05:50.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:50 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1481920223"}]': finished 2024-09-13T23:05:50.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:50 smithi022 ceph-mon[23758]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T23:05:50.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:50 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3419542483' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2315041159"}]: dispatch 2024-09-13T23:05:50.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:50 smithi022 ceph-mon[28440]: pgmap v83: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 639 B/s wr, 130 op/s 2024-09-13T23:05:50.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:50 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1481920223"}]': finished 2024-09-13T23:05:50.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:50 smithi022 ceph-mon[28440]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T23:05:50.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:50 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3419542483' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2315041159"}]: dispatch 2024-09-13T23:05:50.461 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:50 smithi076 ceph-mon[24587]: pgmap v83: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 639 B/s wr, 130 op/s 2024-09-13T23:05:50.461 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:50 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1481920223"}]': finished 2024-09-13T23:05:50.461 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:50 smithi076 ceph-mon[24587]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T23:05:50.461 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:50 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3419542483' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2315041159"}]: dispatch 2024-09-13T23:05:51.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:51 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3419542483' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2315041159"}]': finished 2024-09-13T23:05:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:51 smithi022 ceph-mon[23758]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T23:05:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:51 smithi022 ceph-mon[23758]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 511 B/s wr, 10 op/s 2024-09-13T23:05:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:51 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/162391394' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3245781136"}]: dispatch 2024-09-13T23:05:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:51 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3245781136"}]: dispatch 2024-09-13T23:05:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:51 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3419542483' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2315041159"}]': finished 2024-09-13T23:05:51.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:51 smithi022 ceph-mon[28440]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T23:05:51.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:51 smithi022 ceph-mon[28440]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 511 B/s wr, 10 op/s 2024-09-13T23:05:51.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:51 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/162391394' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3245781136"}]: dispatch 2024-09-13T23:05:51.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:51 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3245781136"}]: dispatch 2024-09-13T23:05:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:51 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3419542483' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2315041159"}]': finished 2024-09-13T23:05:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:51 smithi076 ceph-mon[24587]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T23:05:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:51 smithi076 ceph-mon[24587]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 511 B/s wr, 10 op/s 2024-09-13T23:05:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:51 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/162391394' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3245781136"}]: dispatch 2024-09-13T23:05:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:51 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3245781136"}]: dispatch 2024-09-13T23:05:51.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:05:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:05:51] "GET /metrics HTTP/1.1" 200 207951 "" "Prometheus/2.33.4" 2024-09-13T23:05:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3245781136"}]': finished 2024-09-13T23:05:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T23:05:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2976594162' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/84729394"}]: dispatch 2024-09-13T23:05:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:05:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:05:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: Adding iSCSI gateway http://:@172.21.15.22:5000 to Dashboard 2024-09-13T23:05:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:05:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:05:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:05:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:05:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: Checking dashboard <-> RGW credentials 2024-09-13T23:05:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3245781136"}]': finished 2024-09-13T23:05:52.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T23:05:52.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2976594162' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/84729394"}]: dispatch 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: Adding iSCSI gateway http://:@172.21.15.22:5000 to Dashboard 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:52.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: Checking dashboard <-> RGW credentials 2024-09-13T23:05:52.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3245781136"}]': finished 2024-09-13T23:05:52.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T23:05:52.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2976594162' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/84729394"}]: dispatch 2024-09-13T23:05:52.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:05:52.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:05:52.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: Adding iSCSI gateway http://:@172.21.15.22:5000 to Dashboard 2024-09-13T23:05:52.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:05:52.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:05:52.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:05:52.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:05:52.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:52.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:05:52.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:05:52.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: Checking dashboard <-> RGW credentials 2024-09-13T23:05:52.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:05:53.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:53 smithi022 ceph-mon[23758]: pgmap v88: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 225 B/s wr, 9 op/s; 10 B/s, 0 objects/s recovering 2024-09-13T23:05:53.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:53 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2976594162' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/84729394"}]': finished 2024-09-13T23:05:53.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:53 smithi022 ceph-mon[23758]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T23:05:53.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:53 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1748758242' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/777163445"}]: dispatch 2024-09-13T23:05:53.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:53 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/777163445"}]: dispatch 2024-09-13T23:05:53.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:53 smithi022 ceph-mon[28440]: pgmap v88: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 225 B/s wr, 9 op/s; 10 B/s, 0 objects/s recovering 2024-09-13T23:05:53.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:53 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2976594162' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/84729394"}]': finished 2024-09-13T23:05:53.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:53 smithi022 ceph-mon[28440]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T23:05:53.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:53 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1748758242' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/777163445"}]: dispatch 2024-09-13T23:05:53.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:53 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/777163445"}]: dispatch 2024-09-13T23:05:53.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:53 smithi076 ceph-mon[24587]: pgmap v88: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 225 B/s wr, 9 op/s; 10 B/s, 0 objects/s recovering 2024-09-13T23:05:53.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:53 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2976594162' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/84729394"}]': finished 2024-09-13T23:05:53.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:53 smithi076 ceph-mon[24587]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T23:05:53.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:53 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1748758242' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/777163445"}]: dispatch 2024-09-13T23:05:53.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:53 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/777163445"}]: dispatch 2024-09-13T23:05:54.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:54 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/777163445"}]': finished 2024-09-13T23:05:54.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:54 smithi022 ceph-mon[23758]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T23:05:54.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:54 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/420721120' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1891390217"}]: dispatch 2024-09-13T23:05:54.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:54 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1891390217"}]: dispatch 2024-09-13T23:05:54.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:54 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/777163445"}]': finished 2024-09-13T23:05:54.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:54 smithi022 ceph-mon[28440]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T23:05:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:54 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/420721120' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1891390217"}]: dispatch 2024-09-13T23:05:54.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:54 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1891390217"}]: dispatch 2024-09-13T23:05:54.688 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:54 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/777163445"}]': finished 2024-09-13T23:05:54.688 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:54 smithi076 ceph-mon[24587]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T23:05:54.688 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:54 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/420721120' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1891390217"}]: dispatch 2024-09-13T23:05:54.688 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:54 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1891390217"}]: dispatch 2024-09-13T23:05:55.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:55 smithi076 ceph-mon[24587]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 8.9 KiB/s rd, 246 B/s wr, 11 op/s; 11 B/s, 0 objects/s recovering 2024-09-13T23:05:55.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:55 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1891390217"}]': finished 2024-09-13T23:05:55.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:55 smithi076 ceph-mon[24587]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T23:05:55.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:55 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/4158226896' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4186392471"}]: dispatch 2024-09-13T23:05:55.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:55 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4186392471"}]: dispatch 2024-09-13T23:05:55.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:55 smithi022 ceph-mon[28440]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 8.9 KiB/s rd, 246 B/s wr, 11 op/s; 11 B/s, 0 objects/s recovering 2024-09-13T23:05:55.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:55 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1891390217"}]': finished 2024-09-13T23:05:55.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:55 smithi022 ceph-mon[28440]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T23:05:55.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:55 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/4158226896' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4186392471"}]: dispatch 2024-09-13T23:05:55.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:55 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4186392471"}]: dispatch 2024-09-13T23:05:55.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:55 smithi022 ceph-mon[23758]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 8.9 KiB/s rd, 246 B/s wr, 11 op/s; 11 B/s, 0 objects/s recovering 2024-09-13T23:05:55.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:55 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1891390217"}]': finished 2024-09-13T23:05:55.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:55 smithi022 ceph-mon[23758]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T23:05:55.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:55 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/4158226896' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4186392471"}]: dispatch 2024-09-13T23:05:55.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:55 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4186392471"}]: dispatch 2024-09-13T23:05:56.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:56 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4186392471"}]': finished 2024-09-13T23:05:56.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:56 smithi076 ceph-mon[24587]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T23:05:56.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:56 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2579359229' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3032931192"}]: dispatch 2024-09-13T23:05:56.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:56 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3032931192"}]: dispatch 2024-09-13T23:05:56.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:56 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4186392471"}]': finished 2024-09-13T23:05:56.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:56 smithi022 ceph-mon[23758]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T23:05:56.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:56 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2579359229' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3032931192"}]: dispatch 2024-09-13T23:05:56.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:56 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3032931192"}]: dispatch 2024-09-13T23:05:56.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:56 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4186392471"}]': finished 2024-09-13T23:05:56.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:56 smithi022 ceph-mon[28440]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T23:05:56.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:56 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2579359229' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3032931192"}]: dispatch 2024-09-13T23:05:56.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:56 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3032931192"}]: dispatch 2024-09-13T23:05:57.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:57 smithi076 ceph-mon[24587]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:05:57.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:57 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3032931192"}]': finished 2024-09-13T23:05:57.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:57 smithi076 ceph-mon[24587]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T23:05:57.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:57 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/743718226' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3032931192"}]: dispatch 2024-09-13T23:05:57.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:57 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3032931192"}]: dispatch 2024-09-13T23:05:57.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:57 smithi022 ceph-mon[23758]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:05:57.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:57 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3032931192"}]': finished 2024-09-13T23:05:57.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:57 smithi022 ceph-mon[23758]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T23:05:57.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:57 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/743718226' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3032931192"}]: dispatch 2024-09-13T23:05:57.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:57 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3032931192"}]: dispatch 2024-09-13T23:05:57.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:57 smithi022 ceph-mon[28440]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:05:57.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:57 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3032931192"}]': finished 2024-09-13T23:05:57.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:57 smithi022 ceph-mon[28440]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T23:05:57.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:57 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/743718226' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3032931192"}]: dispatch 2024-09-13T23:05:57.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:57 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3032931192"}]: dispatch 2024-09-13T23:05:58.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:05:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:05:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:05:58.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:58 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3032931192"}]': finished 2024-09-13T23:05:58.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:58 smithi022 ceph-mon[23758]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T23:05:58.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:58 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3558807742' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/84729394"}]: dispatch 2024-09-13T23:05:58.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:58 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/84729394"}]: dispatch 2024-09-13T23:05:58.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:58 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3032931192"}]': finished 2024-09-13T23:05:58.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:58 smithi022 ceph-mon[28440]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T23:05:58.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:58 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3558807742' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/84729394"}]: dispatch 2024-09-13T23:05:58.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:58 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/84729394"}]: dispatch 2024-09-13T23:05:58.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:58 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3032931192"}]': finished 2024-09-13T23:05:58.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:58 smithi076 ceph-mon[24587]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T23:05:58.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:58 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3558807742' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/84729394"}]: dispatch 2024-09-13T23:05:58.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:58 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/84729394"}]: dispatch 2024-09-13T23:05:59.786 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:59 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:05:59.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:59 smithi076 ceph-mon[24587]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:05:59.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:59 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/84729394"}]': finished 2024-09-13T23:05:59.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:59 smithi076 ceph-mon[24587]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T23:05:59.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:59 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/512550590' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/898169442"}]: dispatch 2024-09-13T23:05:59.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:05:59 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/898169442"}]: dispatch 2024-09-13T23:05:59.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:59 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:05:59.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:59 smithi022 ceph-mon[23758]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:05:59.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:59 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/84729394"}]': finished 2024-09-13T23:05:59.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:59 smithi022 ceph-mon[23758]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T23:05:59.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:59 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/512550590' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/898169442"}]: dispatch 2024-09-13T23:05:59.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:05:59 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/898169442"}]: dispatch 2024-09-13T23:05:59.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:59 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:05:59.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:59 smithi022 ceph-mon[28440]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:05:59.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:59 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/84729394"}]': finished 2024-09-13T23:05:59.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:59 smithi022 ceph-mon[28440]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T23:05:59.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:59 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/512550590' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/898169442"}]: dispatch 2024-09-13T23:05:59.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:05:59 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/898169442"}]: dispatch 2024-09-13T23:06:00.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:00 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/898169442"}]': finished 2024-09-13T23:06:00.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:00 smithi076 ceph-mon[24587]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T23:06:00.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:00 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1938142501' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/137268489"}]: dispatch 2024-09-13T23:06:00.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:00 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/137268489"}]: dispatch 2024-09-13T23:06:00.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:00 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/898169442"}]': finished 2024-09-13T23:06:00.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:00 smithi022 ceph-mon[23758]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T23:06:00.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:00 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1938142501' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/137268489"}]: dispatch 2024-09-13T23:06:00.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:00 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/137268489"}]: dispatch 2024-09-13T23:06:00.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:00 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/898169442"}]': finished 2024-09-13T23:06:00.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:00 smithi022 ceph-mon[28440]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T23:06:00.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:00 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1938142501' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/137268489"}]: dispatch 2024-09-13T23:06:00.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:00 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/137268489"}]: dispatch 2024-09-13T23:06:01.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:01 smithi076 ceph-mon[24587]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:01.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:01 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/137268489"}]': finished 2024-09-13T23:06:01.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:01 smithi076 ceph-mon[24587]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T23:06:01.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:01 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2229837622' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1254370233"}]: dispatch 2024-09-13T23:06:01.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:01 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1254370233"}]: dispatch 2024-09-13T23:06:01.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:06:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:06:01] "GET /metrics HTTP/1.1" 200 208083 "" "Prometheus/2.33.4" 2024-09-13T23:06:01.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:01 smithi022 ceph-mon[23758]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:01.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:01 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/137268489"}]': finished 2024-09-13T23:06:01.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:01 smithi022 ceph-mon[23758]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T23:06:01.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:01 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2229837622' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1254370233"}]: dispatch 2024-09-13T23:06:01.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:01 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1254370233"}]: dispatch 2024-09-13T23:06:01.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:01 smithi022 ceph-mon[28440]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:01.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:01 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/137268489"}]': finished 2024-09-13T23:06:01.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:01 smithi022 ceph-mon[28440]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T23:06:01.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:01 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2229837622' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1254370233"}]: dispatch 2024-09-13T23:06:01.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:01 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1254370233"}]: dispatch 2024-09-13T23:06:02.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:02 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1254370233"}]': finished 2024-09-13T23:06:02.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:02 smithi076 ceph-mon[24587]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T23:06:02.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:02 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3870150815' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1724895572"}]: dispatch 2024-09-13T23:06:02.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:02 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1254370233"}]': finished 2024-09-13T23:06:02.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:02 smithi022 ceph-mon[23758]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T23:06:02.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:02 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3870150815' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1724895572"}]: dispatch 2024-09-13T23:06:02.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:02 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1254370233"}]': finished 2024-09-13T23:06:02.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:02 smithi022 ceph-mon[28440]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T23:06:02.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:02 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3870150815' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1724895572"}]: dispatch 2024-09-13T23:06:04.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:03 smithi022 ceph-mon[23758]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:04.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:03 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3870150815' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1724895572"}]': finished 2024-09-13T23:06:04.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:03 smithi022 ceph-mon[23758]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T23:06:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:03 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/194907385' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/137268489"}]: dispatch 2024-09-13T23:06:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:03 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/137268489"}]: dispatch 2024-09-13T23:06:04.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:03 smithi022 ceph-mon[28440]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:04.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:03 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3870150815' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1724895572"}]': finished 2024-09-13T23:06:04.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:03 smithi022 ceph-mon[28440]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T23:06:04.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:03 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/194907385' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/137268489"}]: dispatch 2024-09-13T23:06:04.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:03 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/137268489"}]: dispatch 2024-09-13T23:06:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:03 smithi076 ceph-mon[24587]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:03 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3870150815' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1724895572"}]': finished 2024-09-13T23:06:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:03 smithi076 ceph-mon[24587]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T23:06:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:03 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/194907385' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/137268489"}]: dispatch 2024-09-13T23:06:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:03 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/137268489"}]: dispatch 2024-09-13T23:06:05.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:04 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/137268489"}]': finished 2024-09-13T23:06:05.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:04 smithi022 ceph-mon[23758]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T23:06:05.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:04 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2667025344' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1746885615"}]: dispatch 2024-09-13T23:06:05.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:04 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1746885615"}]: dispatch 2024-09-13T23:06:05.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:04 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/137268489"}]': finished 2024-09-13T23:06:05.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:04 smithi022 ceph-mon[28440]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T23:06:05.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:04 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2667025344' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1746885615"}]: dispatch 2024-09-13T23:06:05.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:04 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1746885615"}]: dispatch 2024-09-13T23:06:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:04 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/137268489"}]': finished 2024-09-13T23:06:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:04 smithi076 ceph-mon[24587]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T23:06:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:04 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2667025344' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1746885615"}]: dispatch 2024-09-13T23:06:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:04 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1746885615"}]: dispatch 2024-09-13T23:06:06.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:05 smithi022 ceph-mon[28440]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:06.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:05 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1746885615"}]': finished 2024-09-13T23:06:06.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:05 smithi022 ceph-mon[28440]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T23:06:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:05 smithi022 ceph-mon[23758]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:05 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1746885615"}]': finished 2024-09-13T23:06:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:05 smithi022 ceph-mon[23758]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T23:06:06.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:05 smithi076 ceph-mon[24587]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:05 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1746885615"}]': finished 2024-09-13T23:06:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:05 smithi076 ceph-mon[24587]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T23:06:08.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:07 smithi022 ceph-mon[23758]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T23:06:08.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:07 smithi022 ceph-mon[28440]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T23:06:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:07 smithi076 ceph-mon[24587]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T23:06:08.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:06:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:06:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:06:10.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:09 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:09 smithi022 ceph-mon[23758]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:09 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:09 smithi022 ceph-mon[28440]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:09 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:09 smithi076 ceph-mon[24587]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:11.810 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:06:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:06:11] "GET /metrics HTTP/1.1" 200 208083 "" "Prometheus/2.33.4" 2024-09-13T23:06:12.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:11 smithi022 ceph-mon[23758]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 678 B/s rd, 0 op/s 2024-09-13T23:06:12.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:11 smithi022 ceph-mon[28440]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 678 B/s rd, 0 op/s 2024-09-13T23:06:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:11 smithi076 ceph-mon[24587]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 678 B/s rd, 0 op/s 2024-09-13T23:06:14.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:13 smithi022 ceph-mon[23758]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:14.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:13 smithi022 ceph-mon[28440]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:14.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:13 smithi076 ceph-mon[24587]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:16.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:15 smithi022 ceph-mon[23758]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:06:16.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:15 smithi022 ceph-mon[28440]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:06:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:15 smithi076 ceph-mon[24587]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:06:18.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:17 smithi022 ceph-mon[23758]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-09-13T23:06:18.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:17 smithi022 ceph-mon[28440]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-09-13T23:06:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:17 smithi076 ceph-mon[24587]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-09-13T23:06:18.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:06:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:06:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:06:20.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:19 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:20.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:19 smithi022 ceph-mon[28440]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:20.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:19 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:20.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:19 smithi022 ceph-mon[23758]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:19 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:19 smithi076 ceph-mon[24587]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:21.824 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:06:21 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:06:21] "GET /metrics HTTP/1.1" 200 208056 "" "Prometheus/2.33.4" 2024-09-13T23:06:22.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:21 smithi022 ceph-mon[23758]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:22.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:21 smithi022 ceph-mon[28440]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:22.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:21 smithi076 ceph-mon[24587]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:24.088 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:23 smithi076 ceph-mon[24587]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:24.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:23 smithi022 ceph-mon[23758]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:24.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:23 smithi022 ceph-mon[28440]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:26.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:25 smithi022 ceph-mon[23758]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:26.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:25 smithi022 ceph-mon[28440]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:25 smithi076 ceph-mon[24587]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:28.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:27 smithi022 ceph-mon[23758]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:28.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:27 smithi022 ceph-mon[28440]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:27 smithi076 ceph-mon[24587]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:28.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:06:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:06:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:06:30.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:29 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:30.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:29 smithi022 ceph-mon[23758]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:30.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:29 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:30.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:29 smithi022 ceph-mon[28440]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:29 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:29 smithi076 ceph-mon[24587]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:31.838 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:06:31 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:06:31] "GET /metrics HTTP/1.1" 200 208016 "" "Prometheus/2.33.4" 2024-09-13T23:06:32.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:31 smithi022 ceph-mon[23758]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:32.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:31 smithi022 ceph-mon[28440]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:32.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:31 smithi076 ceph-mon[24587]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:34.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:33 smithi022 ceph-mon[23758]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:34.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:33 smithi022 ceph-mon[28440]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:33 smithi076 ceph-mon[24587]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:36.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:35 smithi022 ceph-mon[23758]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:36.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:35 smithi022 ceph-mon[28440]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:35 smithi076 ceph-mon[24587]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:37 smithi022 ceph-mon[23758]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:38.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:37 smithi022 ceph-mon[28440]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:38.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:37 smithi076 ceph-mon[24587]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:38.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:06:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:06:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:06:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:39 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:39 smithi022 ceph-mon[23758]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:40.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:39 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:40.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:39 smithi022 ceph-mon[28440]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:39 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:39 smithi076 ceph-mon[24587]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:41.852 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:06:41 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:06:41] "GET /metrics HTTP/1.1" 200 208016 "" "Prometheus/2.33.4" 2024-09-13T23:06:42.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:41 smithi022 ceph-mon[23758]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:42.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:41 smithi022 ceph-mon[28440]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:41 smithi076 ceph-mon[24587]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:44.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:43 smithi022 ceph-mon[23758]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:43 smithi022 ceph-mon[28440]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:44.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:43 smithi076 ceph-mon[24587]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:46.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:45 smithi022 ceph-mon[23758]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:06:46.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:45 smithi022 ceph-mon[28440]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:06:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:45 smithi076 ceph-mon[24587]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:06:47.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:06:47.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:06:47.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:06:47.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:06:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:06:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:06:48.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:47 smithi022 ceph-mon[23758]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:06:48.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:47 smithi022 ceph-mon[28440]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:06:48.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:06:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:06:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:06:48.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:47 smithi076 ceph-mon[24587]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:06:50.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:49 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:50.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:49 smithi022 ceph-mon[28440]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:49 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:49 smithi022 ceph-mon[23758]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:50.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:49 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:06:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:49 smithi076 ceph-mon[24587]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:51.869 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:06:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:06:51] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-09-13T23:06:52.137 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:51 smithi076 ceph-mon[24587]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:06:52.137 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:51 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:06:52.138 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:51 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:06:52.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:51 smithi022 ceph-mon[23758]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:06:52.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:51 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:06:52.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:51 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:06:52.158 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:51 smithi022 ceph-mon[28440]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:06:52.158 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:51 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:06:52.158 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:51 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:06:53.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:53 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:06:53.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:53 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:06:53.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:53 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:06:54.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:54 smithi022 ceph-mon[23758]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:54.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:54 smithi022 ceph-mon[28440]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:54.438 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:54 smithi076 ceph-mon[24587]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:06:56.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:56 smithi022 ceph-mon[23758]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:56 smithi022 ceph-mon[28440]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:56 smithi076 ceph-mon[24587]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:58.173 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:06:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:06:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:06:58.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:06:58 smithi022 ceph-mon[23758]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:58.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:06:58 smithi022 ceph-mon[28440]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:06:58.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:06:58 smithi076 ceph-mon[24587]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:00.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:00 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:00.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:00 smithi022 ceph-mon[23758]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:00.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:00 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:00.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:00 smithi022 ceph-mon[28440]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:00.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:00 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:00 smithi076 ceph-mon[24587]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:01.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:07:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:07:01] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-09-13T23:07:02.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:02 smithi076 ceph-mon[24587]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:02.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:02 smithi022 ceph-mon[23758]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:02.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:02 smithi022 ceph-mon[28440]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:04 smithi076 ceph-mon[24587]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:04.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:04 smithi022 ceph-mon[23758]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:04.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:04 smithi022 ceph-mon[28440]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:06.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:06 smithi076 ceph-mon[24587]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:06.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:06 smithi022 ceph-mon[28440]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:06.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:06 smithi022 ceph-mon[23758]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:08.187 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:07:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:07:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:07:08.478 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:08 smithi022 ceph-mon[23758]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:08.479 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:08 smithi022 ceph-mon[28440]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:08 smithi076 ceph-mon[24587]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:09 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:09 smithi076 ceph-mon[24587]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:09.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:09 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:09.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:09 smithi022 ceph-mon[23758]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:09.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:09 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:09.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:09 smithi022 ceph-mon[28440]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:11.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:11 smithi022 ceph-mon[23758]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:11.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:07:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:07:11] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-09-13T23:07:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:11 smithi022 ceph-mon[28440]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:11.786 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:11 smithi076 ceph-mon[24587]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:14.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:13 smithi022 ceph-mon[23758]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:14.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:13 smithi022 ceph-mon[28440]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:14.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:13 smithi076 ceph-mon[24587]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:16.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:15 smithi022 ceph-mon[23758]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:16.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:15 smithi022 ceph-mon[28440]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:15 smithi076 ceph-mon[24587]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:18.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:17 smithi022 ceph-mon[23758]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:18.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:17 smithi022 ceph-mon[28440]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:17 smithi076 ceph-mon[24587]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:18.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:07:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:07:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:07:20.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:19 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:20.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:19 smithi022 ceph-mon[28440]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:20.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:19 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:20.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:19 smithi022 ceph-mon[23758]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:19 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:19 smithi076 ceph-mon[24587]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:21.814 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:07:21 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:07:21] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-09-13T23:07:22.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:21 smithi022 ceph-mon[23758]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:22.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:21 smithi022 ceph-mon[28440]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:22.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:21 smithi076 ceph-mon[24587]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:24.088 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:23 smithi076 ceph-mon[24587]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:24.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:23 smithi022 ceph-mon[23758]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:24.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:23 smithi022 ceph-mon[28440]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:26.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:25 smithi022 ceph-mon[23758]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:26.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:25 smithi022 ceph-mon[28440]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:26.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:25 smithi076 ceph-mon[24587]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:28.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:27 smithi022 ceph-mon[23758]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:28.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:27 smithi022 ceph-mon[28440]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:27 smithi076 ceph-mon[24587]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:28.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:07:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:07:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:07:30.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:29 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:30.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:29 smithi022 ceph-mon[23758]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:30.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:29 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:30.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:29 smithi022 ceph-mon[28440]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:29 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:29 smithi076 ceph-mon[24587]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:31.828 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:07:31 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:07:31] "GET /metrics HTTP/1.1" 200 207990 "" "Prometheus/2.33.4" 2024-09-13T23:07:32.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:31 smithi022 ceph-mon[23758]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:32.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:31 smithi022 ceph-mon[28440]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:32.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:31 smithi076 ceph-mon[24587]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:34.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:33 smithi022 ceph-mon[23758]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:34.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:33 smithi022 ceph-mon[28440]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:33 smithi076 ceph-mon[24587]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:36.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:35 smithi022 ceph-mon[23758]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:36.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:35 smithi022 ceph-mon[28440]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:36.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:35 smithi076 ceph-mon[24587]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:38.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:37 smithi022 ceph-mon[28440]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:37 smithi022 ceph-mon[23758]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:37 smithi076 ceph-mon[24587]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:38.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:07:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:07:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:07:40.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:39 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:40.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:39 smithi022 ceph-mon[28440]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:39 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:39 smithi022 ceph-mon[23758]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:39 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:39 smithi076 ceph-mon[24587]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:41.847 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:07:41 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:07:41] "GET /metrics HTTP/1.1" 200 207990 "" "Prometheus/2.33.4" 2024-09-13T23:07:42.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:41 smithi022 ceph-mon[28440]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:42.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:41 smithi022 ceph-mon[23758]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:42.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:41 smithi076 ceph-mon[24587]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:44.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:43 smithi022 ceph-mon[23758]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:44.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:43 smithi022 ceph-mon[28440]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:43 smithi076 ceph-mon[24587]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:46.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:45 smithi022 ceph-mon[23758]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:46.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:45 smithi022 ceph-mon[28440]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:46.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:45 smithi076 ceph-mon[24587]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:47.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:07:47.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:07:47.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:07:47.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:07:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:07:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:07:48.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:47 smithi022 ceph-mon[23758]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:48.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:47 smithi022 ceph-mon[28440]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:48.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:47 smithi076 ceph-mon[24587]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:48.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:07:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:07:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:07:50.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:49 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:50.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:49 smithi022 ceph-mon[23758]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:50.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:49 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:50.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:49 smithi022 ceph-mon[28440]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:50.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:49 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:07:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:49 smithi076 ceph-mon[24587]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:51.863 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:07:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:07:51] "GET /metrics HTTP/1.1" 200 207977 "" "Prometheus/2.33.4" 2024-09-13T23:07:52.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:51 smithi022 ceph-mon[23758]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:52.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:51 smithi022 ceph-mon[28440]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:52.199 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:51 smithi076 ceph-mon[24587]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:53.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:07:53.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:07:53.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:07:53.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:07:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:07:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:07:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:07:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:07:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:07:54.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:53 smithi022 ceph-mon[23758]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:54.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:53 smithi022 ceph-mon[28440]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:54.188 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:53 smithi076 ceph-mon[24587]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:07:56.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:55 smithi022 ceph-mon[23758]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:56.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:55 smithi022 ceph-mon[28440]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:55 smithi076 ceph-mon[24587]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:58.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:57 smithi022 ceph-mon[23758]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:58.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:57 smithi022 ceph-mon[28440]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:57 smithi076 ceph-mon[24587]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:07:58.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:07:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:07:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:08:00.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:59 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:00.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:07:59 smithi022 ceph-mon[23758]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:00.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:59 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:00.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:07:59 smithi022 ceph-mon[28440]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:59 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:07:59 smithi076 ceph-mon[24587]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:01.879 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:08:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:08:01] "GET /metrics HTTP/1.1" 200 207978 "" "Prometheus/2.33.4" 2024-09-13T23:08:02.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:01 smithi022 ceph-mon[23758]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:02.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:01 smithi022 ceph-mon[28440]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:02.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:01 smithi076 ceph-mon[24587]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:04.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:03 smithi022 ceph-mon[28440]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:03 smithi022 ceph-mon[23758]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:04.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:03 smithi076 ceph-mon[24587]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:06.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:05 smithi022 ceph-mon[28440]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:06.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:05 smithi022 ceph-mon[23758]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:05 smithi076 ceph-mon[24587]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:08.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:07 smithi022 ceph-mon[28440]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:08.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:07 smithi022 ceph-mon[23758]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:07 smithi076 ceph-mon[24587]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:08.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:08:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:08:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:08:10.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:09 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:09 smithi022 ceph-mon[28440]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:09 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:09 smithi022 ceph-mon[23758]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:09 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:09 smithi076 ceph-mon[24587]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:11.893 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:08:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:08:11] "GET /metrics HTTP/1.1" 200 207978 "" "Prometheus/2.33.4" 2024-09-13T23:08:12.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:11 smithi022 ceph-mon[23758]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:12.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:11 smithi022 ceph-mon[28440]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:12.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:11 smithi076 ceph-mon[24587]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:14.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:13 smithi022 ceph-mon[23758]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:14.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:13 smithi022 ceph-mon[28440]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:14.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:13 smithi076 ceph-mon[24587]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:16.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:15 smithi022 ceph-mon[23758]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:16.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:15 smithi022 ceph-mon[28440]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:16.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:15 smithi076 ceph-mon[24587]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:18.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:17 smithi022 ceph-mon[23758]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:18.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:17 smithi022 ceph-mon[28440]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:17 smithi076 ceph-mon[24587]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:18.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:08:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:08:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:08:20.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:19 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:20.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:19 smithi022 ceph-mon[23758]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:19 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:19 smithi022 ceph-mon[28440]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:19 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:19 smithi076 ceph-mon[24587]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:21.907 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:08:21 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:08:21] "GET /metrics HTTP/1.1" 200 207938 "" "Prometheus/2.33.4" 2024-09-13T23:08:22.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:21 smithi022 ceph-mon[23758]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:08:22.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:21 smithi022 ceph-mon[28440]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:08:22.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:21 smithi076 ceph-mon[24587]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:08:24.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:23 smithi022 ceph-mon[23758]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:24.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:23 smithi022 ceph-mon[28440]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:24.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:23 smithi076 ceph-mon[24587]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:26.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:25 smithi022 ceph-mon[23758]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:08:26.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:25 smithi022 ceph-mon[28440]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:08:26.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:25 smithi076 ceph-mon[24587]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:08:28.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:27 smithi022 ceph-mon[23758]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:28.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:27 smithi022 ceph-mon[28440]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:28.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:27 smithi076 ceph-mon[24587]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:28.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:08:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:08:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:08:30.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:29 smithi022 ceph-mon[28440]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:30.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:29 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:30.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:29 smithi022 ceph-mon[23758]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:30.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:29 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:29 smithi076 ceph-mon[24587]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:29 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:31.921 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:08:31 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:08:31] "GET /metrics HTTP/1.1" 200 207936 "" "Prometheus/2.33.4" 2024-09-13T23:08:32.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:31 smithi022 ceph-mon[28440]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:32.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:31 smithi022 ceph-mon[23758]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:32.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:31 smithi076 ceph-mon[24587]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:34.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:33 smithi022 ceph-mon[28440]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:34.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:33 smithi022 ceph-mon[23758]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:33 smithi076 ceph-mon[24587]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:35 smithi076 ceph-mon[24587]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:36.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:35 smithi022 ceph-mon[28440]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:36.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:35 smithi022 ceph-mon[23758]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:38.228 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:37 smithi022 ceph-mon[23758]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:38.229 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:37 smithi022 ceph-mon[28440]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:37 smithi076 ceph-mon[24587]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:38.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:08:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:08:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:08:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:39 smithi076 ceph-mon[24587]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:39 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:40.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:39 smithi022 ceph-mon[23758]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:40.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:39 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:40.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:39 smithi022 ceph-mon[28440]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:40.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:39 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:41.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:08:41 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:08:41] "GET /metrics HTTP/1.1" 200 207936 "" "Prometheus/2.33.4" 2024-09-13T23:08:42.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:41 smithi076 ceph-mon[24587]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:42.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:41 smithi022 ceph-mon[23758]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:42.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:41 smithi022 ceph-mon[28440]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:43 smithi076 ceph-mon[24587]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:44.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:43 smithi022 ceph-mon[23758]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:44.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:43 smithi022 ceph-mon[28440]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:46.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:45 smithi022 ceph-mon[23758]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:46.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:45 smithi022 ceph-mon[28440]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:46.249 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-13T23:08:46.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:45 smithi076 ceph-mon[24587]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:47.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:08:47.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:08:47.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:08:47.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:08:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:08:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:08:48.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:47 smithi022 ceph-mon[23758]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:48.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:47 smithi022 ceph-mon[28440]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:48.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:47 smithi076 ceph-mon[24587]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:48.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:08:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:08:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:08:48.967 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-13T23:08:50.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:49 smithi076 ceph-mon[24587]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:49 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:50.301 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:49 smithi022 ceph-mon[23758]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:50.301 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:49 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:50.301 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:49 smithi022 ceph-mon[28440]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:50.302 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:49 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:08:51.601 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:08:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:08:51] "GET /metrics HTTP/1.1" 200 207931 "" "Prometheus/2.33.4" 2024-09-13T23:08:51.628 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-13T23:08:52.279 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:51 smithi022 ceph-mon[23758]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:52.279 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:51 smithi022 ceph-mon[28440]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:52.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:51 smithi076 ceph-mon[24587]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:53.205 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:08:53.206 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:52 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:08:53.206 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:08:53.206 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:52 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:08:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:08:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:52 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:08:54.234 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:53 smithi022 ceph-mon[28440]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:54.234 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:53 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:08:54.234 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:53 smithi022 ceph-mon[23758]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:54.234 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:53 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:08:54.258 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:08:54.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:53 smithi076 ceph-mon[24587]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:08:54.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:53 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:08:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:55 smithi076 ceph-mon[24587]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:56.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:55 smithi022 ceph-mon[23758]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:56.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:55 smithi022 ceph-mon[28440]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:56.436 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:08:56.436 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (3m) 3m ago 4m 14.6M - ba2b418f427c 863a54ec4b87 2024-09-13T23:08:56.436 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (3m) 3m ago 3m 42.8M - 8.3.5 dad864ee21e9 c818389ee4e7 2024-09-13T23:08:56.436 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (3m) 3m ago 3m 64.8M - 3.5 e1d6a67b021e 0dca3d2f62a7 2024-09-13T23:08:56.436 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e e3643f699b35 2024-09-13T23:08:56.436 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:9283 running (10m) 3m ago 10m 455M - 17.2.0 e1d6a67b021e 07f20d2b59b7 2024-09-13T23:08:56.436 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (10m) 3m ago 10m 44.3M 2048M 17.2.0 e1d6a67b021e 7d59bd9c71e8 2024-09-13T23:08:56.436 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (9m) 3m ago 8m 38.3M 2048M 17.2.0 e1d6a67b021e a1194b4b1c43 2024-09-13T23:08:56.437 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (9m) 3m ago 9m 36.2M 2048M 17.2.0 e1d6a67b021e 336a6f00a2a7 2024-09-13T23:08:56.437 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (4m) 3m ago 4m 17.1M - 1dbe0e931976 68ffcaba64d7 2024-09-13T23:08:56.437 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (4m) 3m ago 4m 18.1M - 1dbe0e931976 6ee1b8ce1803 2024-09-13T23:08:56.437 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (8m) 3m ago 8m 46.5M 4017M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:08:56.437 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (8m) 3m ago 8m 48.7M 4017M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:08:56.437 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (7m) 3m ago 7m 44.8M 4017M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:08:56.437 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (7m) 3m ago 7m 44.6M 4017M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:08:56.437 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (6m) 3m ago 6m 47.0M 4272M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:08:56.437 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (6m) 3m ago 6m 47.7M 4272M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:08:56.437 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (6m) 3m ago 6m 46.6M 4272M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:08:56.437 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (5m) 3m ago 5m 45.8M 4272M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:08:56.438 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (3m) 3m ago 4m 44.6M - 514e6a882f6e 8e9cd8b611e3 2024-09-13T23:08:56.438 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (3m) 3m ago 3m 70.2M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:08:56.438 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (3m) 3m ago 3m 67.3M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:08:57.273 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T23:08:58.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:57 smithi022 ceph-mon[23758]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:58.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:57 smithi022 ceph-mon[23758]: from='client.24889 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:08:58.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:57 smithi022 ceph-mon[28440]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:58.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:57 smithi022 ceph-mon[28440]: from='client.24889 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:08:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:57 smithi076 ceph-mon[24587]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:08:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:57 smithi076 ceph-mon[24587]: from='client.24889 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:08:58.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:08:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:08:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:08:59.443 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:08:59.443 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:08:59.444 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T23:08:59.444 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:08:59.444 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:08:59.444 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:08:59.444 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:08:59.444 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:08:59.444 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T23:08:59.444 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:08:59.444 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": {}, 2024-09-13T23:08:59.444 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:08:59.444 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:08:59.445 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:08:59.445 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:08:59.445 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-09-13T23:08:59.445 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:08:59.445 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:09:00.055 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:59 smithi022 ceph-mon[23758]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:00.055 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:59 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:00.055 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:08:59 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2843715399' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:09:00.055 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:59 smithi022 ceph-mon[28440]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:00.055 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:59 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:00.055 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:08:59 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2843715399' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:09:00.082 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph -s' 2024-09-13T23:09:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:59 smithi076 ceph-mon[24587]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:59 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:08:59 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2843715399' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:09:01.627 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:09:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:09:01] "GET /metrics HTTP/1.1" 200 207922 "" "Prometheus/2.33.4" 2024-09-13T23:09:02.100 INFO:teuthology.orchestra.run.smithi022.stdout: cluster: 2024-09-13T23:09:02.100 INFO:teuthology.orchestra.run.smithi022.stdout: id: 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:09:02.100 INFO:teuthology.orchestra.run.smithi022.stdout: health: HEALTH_OK 2024-09-13T23:09:02.100 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:09:02.100 INFO:teuthology.orchestra.run.smithi022.stdout: services: 2024-09-13T23:09:02.100 INFO:teuthology.orchestra.run.smithi022.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-09-13T23:09:02.100 INFO:teuthology.orchestra.run.smithi022.stdout: mgr: y(active, since 5m), standbys: x 2024-09-13T23:09:02.100 INFO:teuthology.orchestra.run.smithi022.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-09-13T23:09:02.101 INFO:teuthology.orchestra.run.smithi022.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-13T23:09:02.101 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:09:02.101 INFO:teuthology.orchestra.run.smithi022.stdout: data: 2024-09-13T23:09:02.101 INFO:teuthology.orchestra.run.smithi022.stdout: pools: 6 pools, 161 pgs 2024-09-13T23:09:02.101 INFO:teuthology.orchestra.run.smithi022.stdout: objects: 209 objects, 585 KiB 2024-09-13T23:09:02.101 INFO:teuthology.orchestra.run.smithi022.stdout: usage: 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:09:02.101 INFO:teuthology.orchestra.run.smithi022.stdout: pgs: 161 active+clean 2024-09-13T23:09:02.101 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:09:02.101 INFO:teuthology.orchestra.run.smithi022.stdout: io: 2024-09-13T23:09:02.101 INFO:teuthology.orchestra.run.smithi022.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-13T23:09:02.101 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:09:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:01 smithi076 ceph-mon[24587]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:02.419 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:01 smithi022 ceph-mon[28440]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:02.420 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:01 smithi022 ceph-mon[23758]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:02.764 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ls' 2024-09-13T23:09:03.079 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:02 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/780139115' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T23:09:03.079 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:02 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/780139115' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T23:09:03.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:02 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/780139115' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T23:09:04.086 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:03 smithi022 ceph-mon[28440]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:04.086 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:03 smithi022 ceph-mon[23758]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:03 smithi076 ceph-mon[24587]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:04.787 INFO:teuthology.orchestra.run.smithi022.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-13T23:09:04.787 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi022=a;count:1 2024-09-13T23:09:04.787 INFO:teuthology.orchestra.run.smithi022.stdout:grafana ?:3000 1/1 3m ago 5m smithi076=a;count:1 2024-09-13T23:09:04.787 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-09-13T23:09:04.788 INFO:teuthology.orchestra.run.smithi022.stdout:mgr 2/2 3m ago 8m smithi022=y;smithi076=x;count:2 2024-09-13T23:09:04.788 INFO:teuthology.orchestra.run.smithi022.stdout:mon 3/3 3m ago 9m smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi076:172.21.15.76=b;count:3 2024-09-13T23:09:04.788 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi022=a;smithi076=b;count:2 2024-09-13T23:09:04.788 INFO:teuthology.orchestra.run.smithi022.stdout:osd 8 3m ago - 2024-09-13T23:09:04.788 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus ?:9095 1/1 3m ago 5m smithi076=a;count:1 2024-09-13T23:09:04.788 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-09-13T23:09:05.487 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T23:09:06.131 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:05 smithi022 ceph-mon[28440]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:06.131 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:05 smithi022 ceph-mon[28440]: from='client.15003 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:09:06.131 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:05 smithi022 ceph-mon[23758]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:06.132 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:05 smithi022 ceph-mon[23758]: from='client.15003 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:09:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:05 smithi076 ceph-mon[24587]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:05 smithi076 ceph-mon[24587]: from='client.15003 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:09:07.842 INFO:teuthology.orchestra.run.smithi022.stdout:Scheduled to redeploy mgr.x on host 'smithi076' 2024-09-13T23:09:08.136 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:07 smithi022 ceph-mon[23758]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:08.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:07 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2787011760' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T23:09:08.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:07 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:08.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:07 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:09:08.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:07 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:09:08.137 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:07 smithi022 ceph-mon[28440]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:08.137 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:07 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2787011760' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T23:09:08.137 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:07 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:08.137 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:07 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:09:08.137 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:07 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:09:08.181 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:09:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:09:08.181 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:07 smithi076 ceph-mon[24587]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:08.181 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:07 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2787011760' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T23:09:08.181 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:07 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:08.181 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:07 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:09:08.181 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:07 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:09:08.558 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps --refresh' 2024-09-13T23:09:09.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[23758]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:09:09.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[23758]: Schedule redeploy daemon mgr.x 2024-09-13T23:09:09.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:09.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:09.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:09:09.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:09:09.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:09:09.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[23758]: Deploying daemon mgr.x on smithi076 2024-09-13T23:09:09.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[28440]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:09:09.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[28440]: Schedule redeploy daemon mgr.x 2024-09-13T23:09:09.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:09.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:09.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:09:09.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:09:09.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:09:09.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:09 smithi022 ceph-mon[28440]: Deploying daemon mgr.x on smithi076 2024-09-13T23:09:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:09 smithi076 ceph-mon[24587]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:09:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:09 smithi076 ceph-mon[24587]: Schedule redeploy daemon mgr.x 2024-09-13T23:09:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:09 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:09 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:09 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:09:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:09 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:09:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:09 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:09:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:09 smithi076 ceph-mon[24587]: Deploying daemon mgr.x on smithi076 2024-09-13T23:09:10.420 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:10 smithi022 ceph-mon[23758]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:10.420 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:10 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:10.421 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:10 smithi022 ceph-mon[28440]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:10.421 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:10 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:10.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:10 smithi076 ceph-mon[24587]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:10.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:10 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:10.621 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:09:10.621 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (3m) 3m ago 4m 14.6M - ba2b418f427c 863a54ec4b87 2024-09-13T23:09:10.621 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (4m) 3m ago 4m 42.8M - 8.3.5 dad864ee21e9 c818389ee4e7 2024-09-13T23:09:10.621 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (3m) 3m ago 3m 64.8M - 3.5 e1d6a67b021e 0dca3d2f62a7 2024-09-13T23:09:10.621 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e e3643f699b35 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:9283 running (10m) 3m ago 10m 455M - 17.2.0 e1d6a67b021e 07f20d2b59b7 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (10m) 3m ago 10m 44.3M 2048M 17.2.0 e1d6a67b021e 7d59bd9c71e8 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (9m) 3m ago 9m 38.3M 2048M 17.2.0 e1d6a67b021e a1194b4b1c43 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (9m) 3m ago 9m 36.2M 2048M 17.2.0 e1d6a67b021e 336a6f00a2a7 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (5m) 3m ago 5m 17.1M - 1dbe0e931976 68ffcaba64d7 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (4m) 3m ago 4m 18.1M - 1dbe0e931976 6ee1b8ce1803 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (8m) 3m ago 8m 46.5M 4017M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (8m) 3m ago 8m 48.7M 4017M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (7m) 3m ago 7m 44.8M 4017M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (7m) 3m ago 7m 44.6M 4017M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (7m) 3m ago 7m 47.0M 4272M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (6m) 3m ago 6m 47.7M 4272M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (6m) 3m ago 6m 46.6M 4272M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:09:10.622 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (5m) 3m ago 5m 45.8M 4272M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:09:10.623 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (3m) 3m ago 5m 44.6M - 514e6a882f6e 8e9cd8b611e3 2024-09-13T23:09:10.623 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (3m) 3m ago 3m 70.2M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:09:10.623 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (3m) 3m ago 3m 67.3M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:09:11.397 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 180' 2024-09-13T23:09:11.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:09:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:09:11] "GET /metrics HTTP/1.1" 200 207922 "" "Prometheus/2.33.4" 2024-09-13T23:09:12.374 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:12 smithi022 ceph-mon[23758]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:12.374 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:12 smithi022 ceph-mon[23758]: from='client.24925 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:09:12.374 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:12 smithi022 ceph-mon[28440]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:12.374 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:12 smithi022 ceph-mon[28440]: from='client.24925 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:09:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:12 smithi076 ceph-mon[24587]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:12 smithi076 ceph-mon[24587]: from='client.24925 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:09:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:14 smithi022 ceph-mon[23758]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:14 smithi022 ceph-mon[28440]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:14.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:14 smithi076 ceph-mon[24587]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:16.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:16 smithi022 ceph-mon[23758]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:16.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:16 smithi022 ceph-mon[28440]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:16.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:16 smithi076 ceph-mon[24587]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:18.159 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:09:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:09:18.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:18 smithi022 ceph-mon[23758]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:18.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:18 smithi022 ceph-mon[28440]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:18.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:18 smithi076 ceph-mon[24587]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:20.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:20 smithi022 ceph-mon[23758]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:20.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:20 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:20.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:20 smithi022 ceph-mon[28440]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:20.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:20 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:20.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:20 smithi076 ceph-mon[24587]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:20.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:20 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:21.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:09:21 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:09:21] "GET /metrics HTTP/1.1" 200 207926 "" "Prometheus/2.33.4" 2024-09-13T23:09:22.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:22 smithi022 ceph-mon[23758]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:22 smithi022 ceph-mon[28440]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:22.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:22 smithi076 ceph-mon[24587]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:24.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:24 smithi022 ceph-mon[28440]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:24.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:24 smithi022 ceph-mon[23758]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:24.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:24 smithi076 ceph-mon[24587]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:26.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:26 smithi022 ceph-mon[28440]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:26.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:26 smithi022 ceph-mon[23758]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:26.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:26 smithi076 ceph-mon[24587]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:28.170 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[25749]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:09:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:09:28.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:28 smithi022 ceph-mon[28440]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:28.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:28 smithi022 ceph-mon[23758]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:28.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:28 smithi076 ceph-mon[24587]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:30.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:30 smithi022 ceph-mon[23758]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:30.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:30 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:30.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:30 smithi022 ceph-mon[28440]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:30.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:30 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:30 smithi076 ceph-mon[24587]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:30 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:31.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:09:31 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:09:31] "GET /metrics HTTP/1.1" 200 207934 "" "Prometheus/2.33.4" 2024-09-13T23:09:32.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:32 smithi022 ceph-mon[23758]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:32.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:32 smithi022 ceph-mon[28440]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:32.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:32 smithi076 ceph-mon[24587]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:34.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:34 smithi076 ceph-mon[24587]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:34.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:34 smithi022 ceph-mon[28440]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:34.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:34 smithi022 ceph-mon[23758]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:36 smithi076 ceph-mon[24587]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:36.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:36 smithi076 systemd[1]: Stopping Ceph mgr.x for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:09:36.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:36 smithi076 bash[44104]: Error: no container with name or ID "ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr.x" found: no such container 2024-09-13T23:09:36.618 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:36 smithi076 podman[44115]: 2024-09-13 23:09:36.3561167 +0000 UTC m=+0.133996506 container died e3643f699b3582111e1702b9ae966899b2081cef4ccac8c15a743ae41307b934 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, io.openshift.tags=base centos centos-stream, ceph=True, GIT_CLEAN=True, RELEASE=HEAD, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, vcs-type=git, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T23:09:36.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:36 smithi022 ceph-mon[28440]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:36.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:36 smithi022 ceph-mon[23758]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:36.993 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:36 smithi076 podman[44115]: 2024-09-13 23:09:36.700043964 +0000 UTC m=+0.477923770 container cleanup e3643f699b3582111e1702b9ae966899b2081cef4ccac8c15a743ae41307b934 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, architecture=x86_64, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.expose-services=, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, distribution-scope=public, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0) 2024-09-13T23:09:36.993 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:36 smithi076 bash[44115]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x 2024-09-13T23:09:36.993 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:36 smithi076 bash[44141]: Error: no container with name or ID "ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr.x" found: no such container 2024-09-13T23:09:37.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:36 smithi076 podman[44127]: 2024-09-13 23:09:36.992835165 +0000 UTC m=+0.636520261 container remove e3643f699b3582111e1702b9ae966899b2081cef4ccac8c15a743ae41307b934 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, distribution-scope=public, vendor=Red Hat, Inc., version=8, io.k8s.display-name=CentOS Stream 8, release=754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, architecture=x86_64, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.openshift.expose-services=, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0) 2024-09-13T23:09:37.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:36 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T23:09:37.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:36 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T23:09:37.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:36 smithi076 systemd[1]: Stopped Ceph mgr.x for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:09:37.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:36 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.x.service: Consumed 17.331s CPU time. 2024-09-13T23:09:37.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:37 smithi076 systemd[1]: Starting Ceph mgr.x for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:09:37.680 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:37 smithi076 podman[44242]: 2024-09-13 23:09:37.32646484 +0000 UTC m=+0.016319554 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:09:37.680 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:37 smithi076 podman[44242]: 2024-09-13 23:09:37.427201598 +0000 UTC m=+0.117056287 container create 194c1a9a8cc9fcc052fc89adbbae3261eeb35cee1dba6eca226117dfb6132bf3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-13T23:09:37.680 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:37 smithi076 podman[44242]: 2024-09-13 23:09:37.679913229 +0000 UTC m=+0.369767919 container init 194c1a9a8cc9fcc052fc89adbbae3261eeb35cee1dba6eca226117dfb6132bf3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T23:09:38.032 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:37 smithi076 podman[44242]: 2024-09-13 23:09:37.68314047 +0000 UTC m=+0.372995170 container start 194c1a9a8cc9fcc052fc89adbbae3261eeb35cee1dba6eca226117dfb6132bf3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-13T23:09:38.032 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:37 smithi076 ceph-mgr[44261]: -- 172.21.15.76:0/56400960 <== mon.2 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bf2037f0e0 con 0x55bf2035d400 2024-09-13T23:09:38.032 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:37 smithi076 bash[44242]: 194c1a9a8cc9fcc052fc89adbbae3261eeb35cee1dba6eca226117dfb6132bf3 2024-09-13T23:09:38.032 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:37 smithi076 systemd[1]: Started Ceph mgr.x for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:09:38.032 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:37.807+0000 7f40ae664040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T23:09:38.032 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:37.870+0000 7f40ae664040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T23:09:38.032 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:38.030+0000 7f40ae664040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T23:09:38.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:38.105+0000 7f40ae664040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T23:09:38.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:38.265+0000 7f40ae664040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:09:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:38 smithi076 ceph-mon[24587]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:38 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:38 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:38 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:09:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:38 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:09:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:38 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:38.451 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:38 smithi022 ceph-mon[23758]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:38.451 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:38 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:38.451 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:38 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:38.452 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:38 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:09:38.452 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:38 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:09:38.452 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:38 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:38.452 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:38 smithi022 ceph-mon[28440]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:38.452 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:38 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:38.452 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:38 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:38.452 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:38 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:09:38.452 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:38 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:09:38.452 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:38 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:38.748 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:38.423+0000 7f40ae664040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T23:09:39.012 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:38.753+0000 7f40ae664040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T23:09:39.012 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:38.817+0000 7f40ae664040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T23:09:39.012 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:39.010+0000 7f40ae664040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T23:09:40.281 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:40 smithi076 ceph-mon[24587]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:40.281 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:40 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:40.282 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:40 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:40.017+0000 7f40ae664040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T23:09:40.282 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:40 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:40.093+0000 7f40ae664040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T23:09:40.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:40 smithi022 ceph-mon[23758]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:40.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:40 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:40.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:40 smithi022 ceph-mon[28440]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:40.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:40 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:41.144 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:40 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:40.870+0000 7f40ae664040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T23:09:41.145 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:40 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:40.989+0000 7f40ae664040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T23:09:41.434 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:41.143+0000 7f40ae664040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:09:41.435 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:41.207+0000 7f40ae664040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T23:09:41.435 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:41.362+0000 7f40ae664040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T23:09:41.435 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:41.433+0000 7f40ae664040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T23:09:41.734 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:41.496+0000 7f40ae664040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T23:09:41.735 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:41.667+0000 7f40ae664040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T23:09:41.735 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:41.733+0000 7f40ae664040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T23:09:41.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:09:41 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:09:41] "GET /metrics HTTP/1.1" 200 207934 "" "Prometheus/2.33.4" 2024-09-13T23:09:42.005 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:41.810+0000 7f40ae664040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T23:09:42.005 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:41.898+0000 7f40ae664040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T23:09:42.005 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:41.959+0000 7f40ae664040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T23:09:42.257 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:42.021+0000 7f40ae664040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T23:09:42.257 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:42 smithi076 ceph-mon[24587]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:42.257 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:42 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:42.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:42 smithi022 ceph-mon[23758]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:42.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:42 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:42.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:42 smithi022 ceph-mon[28440]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:42.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:42 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:43.267 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:42.831+0000 7f40ae664040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T23:09:43.267 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:09:42] ENGINE Bus STARTING 2024-09-13T23:09:43.267 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: CherryPy Checker: 2024-09-13T23:09:43.267 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: The Application mounted at '' has an empty config. 2024-09-13T23:09:43.267 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:09:43.267 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:09:42] ENGINE Serving on http://:::9283 2024-09-13T23:09:43.267 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:09:42] ENGINE Bus STARTED 2024-09-13T23:09:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:43 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:43 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:43 smithi076 ceph-mon[24587]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:43 smithi076 ceph-mon[24587]: Standby manager daemon x restarted 2024-09-13T23:09:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:43 smithi076 ceph-mon[24587]: Standby manager daemon x started 2024-09-13T23:09:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:43 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:09:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:43 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:09:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:43 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:09:43.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:43 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:09:43.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:43.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:43.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[23758]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:43.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[23758]: Standby manager daemon x restarted 2024-09-13T23:09:43.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[23758]: Standby manager daemon x started 2024-09-13T23:09:43.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:09:43.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:09:43.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:09:43.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:09:43.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:43.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:09:43.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[28440]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:43.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[28440]: Standby manager daemon x restarted 2024-09-13T23:09:43.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[28440]: Standby manager daemon x started 2024-09-13T23:09:43.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:09:43.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:09:43.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:09:43.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:43 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:09:44.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:44 smithi076 ceph-mon[24587]: mgrmap e22: y(active, since 5m), standbys: x 2024-09-13T23:09:44.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:44 smithi022 ceph-mon[23758]: mgrmap e22: y(active, since 5m), standbys: x 2024-09-13T23:09:44.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:44 smithi022 ceph-mon[28440]: mgrmap e22: y(active, since 5m), standbys: x 2024-09-13T23:09:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:45 smithi076 ceph-mon[24587]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:45.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:45 smithi022 ceph-mon[23758]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:45.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:45 smithi022 ceph-mon[28440]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:46.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:09:46.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:09:46.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:09:46.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:09:46.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:09:46.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:09:48.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:47 smithi022 ceph-mon[23758]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:48.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:47 smithi022 ceph-mon[28440]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:48.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:47 smithi076 ceph-mon[24587]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:48.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:09:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:09:50.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:49 smithi022 ceph-mon[28440]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:50.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:49 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:49 smithi022 ceph-mon[23758]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:49 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:49 smithi076 ceph-mon[24587]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:49 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:09:51.842 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:09:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:09:51] "GET /metrics HTTP/1.1" 200 207944 "" "Prometheus/2.33.4" 2024-09-13T23:09:52.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:51 smithi022 ceph-mon[23758]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:52.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:51 smithi022 ceph-mon[28440]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:52.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:51 smithi076 ceph-mon[24587]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:53.845 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:09:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:09:53.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:09:53.845 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:09:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:09:53.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:09:54.104 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:53 smithi076 ceph-mon[24587]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:54.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:53 smithi022 ceph-mon[23758]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:54.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:53 smithi022 ceph-mon[28440]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:09:56.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:55 smithi022 ceph-mon[23758]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:56.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:55 smithi022 ceph-mon[28440]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:55 smithi076 ceph-mon[24587]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:58.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:57 smithi022 ceph-mon[23758]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:58.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:57 smithi022 ceph-mon[28440]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:09:58.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:09:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:09:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:09:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:57 smithi076 ceph-mon[24587]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:00.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:59 smithi022 ceph-mon[28440]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:00.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:09:59 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:00.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:59 smithi022 ceph-mon[23758]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:00.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:09:59 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:00.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:59 smithi076 ceph-mon[24587]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:09:59 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:01.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:00 smithi022 ceph-mon[28440]: overall HEALTH_OK 2024-09-13T23:10:01.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:00 smithi022 ceph-mon[23758]: overall HEALTH_OK 2024-09-13T23:10:01.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:00 smithi076 ceph-mon[24587]: overall HEALTH_OK 2024-09-13T23:10:01.859 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:10:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:10:01] "GET /metrics HTTP/1.1" 200 207944 "" "Prometheus/2.33.4" 2024-09-13T23:10:02.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:01 smithi022 ceph-mon[28440]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:02.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:01 smithi022 ceph-mon[23758]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:02.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:01 smithi076 ceph-mon[24587]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:03.862 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:10:03.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:10:03.863 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:10:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:10:03.863 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:10:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:10:04.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:03 smithi022 ceph-mon[23758]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:04.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:03 smithi022 ceph-mon[28440]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:03 smithi076 ceph-mon[24587]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:06.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:05 smithi022 ceph-mon[23758]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:06.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:05 smithi022 ceph-mon[28440]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:05 smithi076 ceph-mon[24587]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:08.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:07 smithi022 ceph-mon[23758]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:07 smithi022 ceph-mon[28440]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:07 smithi076 ceph-mon[24587]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:08.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:10:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:10:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:10:10.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:09 smithi022 ceph-mon[23758]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:09 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:09 smithi022 ceph-mon[28440]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:09 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:09 smithi076 ceph-mon[24587]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:09 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:11.873 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:10:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:10:11] "GET /metrics HTTP/1.1" 200 207944 "" "Prometheus/2.33.4" 2024-09-13T23:10:12.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:11 smithi022 ceph-mon[23758]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:12.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:11 smithi022 ceph-mon[28440]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:12.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:11 smithi076 ceph-mon[24587]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:13.876 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:10:13.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:10:13.877 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:10:13.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:10:13.877 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:10:13.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:10:14.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:13 smithi022 ceph-mon[23758]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:14.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:13 smithi022 ceph-mon[28440]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:14.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:13 smithi076 ceph-mon[24587]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:16.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:15 smithi022 ceph-mon[23758]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:16.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:15 smithi022 ceph-mon[28440]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:15 smithi076 ceph-mon[24587]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:18.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:17 smithi022 ceph-mon[23758]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:18.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:17 smithi022 ceph-mon[28440]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:17 smithi076 ceph-mon[24587]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:18.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:10:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:10:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:10:19.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:19 smithi022 ceph-mon[28440]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:19.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:19 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:19.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:19 smithi022 ceph-mon[23758]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:19.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:19 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:19 smithi076 ceph-mon[24587]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:19 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:21.887 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:10:21 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:10:21] "GET /metrics HTTP/1.1" 200 207948 "" "Prometheus/2.33.4" 2024-09-13T23:10:22.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:21 smithi022 ceph-mon[23758]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:22.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:21 smithi022 ceph-mon[28440]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:22.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:21 smithi076 ceph-mon[24587]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:23.890 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:23 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:10:23.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:10:23.891 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:23 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:10:23.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:10:23.891 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:23 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:10:23.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:10:24.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:23 smithi022 ceph-mon[23758]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:24.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:23 smithi022 ceph-mon[28440]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:24.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:23 smithi076 ceph-mon[24587]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:26.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:25 smithi022 ceph-mon[23758]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:26.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:25 smithi022 ceph-mon[28440]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:25 smithi076 ceph-mon[24587]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:28.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:27 smithi022 ceph-mon[23758]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:28.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:27 smithi022 ceph-mon[28440]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:27 smithi076 ceph-mon[24587]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:28.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:10:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:10:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:10:30.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:29 smithi022 ceph-mon[23758]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:30.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:29 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:30.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:29 smithi022 ceph-mon[28440]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:30.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:29 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:30.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:29 smithi076 ceph-mon[24587]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:30.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:29 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:31.902 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:10:31 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:10:31] "GET /metrics HTTP/1.1" 200 207949 "" "Prometheus/2.33.4" 2024-09-13T23:10:32.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:31 smithi022 ceph-mon[23758]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:32.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:31 smithi022 ceph-mon[28440]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:32.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:31 smithi076 ceph-mon[24587]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:33.906 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:10:33.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:10:33.906 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:10:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:10:33.906 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:10:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:10:34.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:33 smithi022 ceph-mon[23758]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:34.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:33 smithi022 ceph-mon[28440]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:33 smithi076 ceph-mon[24587]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:36.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:35 smithi022 ceph-mon[28440]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:36.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:35 smithi022 ceph-mon[23758]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:35 smithi076 ceph-mon[24587]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:37 smithi022 ceph-mon[23758]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:38.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:37 smithi022 ceph-mon[28440]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:37 smithi076 ceph-mon[24587]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:38.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:10:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:10:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:10:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:39 smithi022 ceph-mon[23758]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:39 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:40.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:39 smithi022 ceph-mon[28440]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:40.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:39 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:39 smithi076 ceph-mon[24587]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:39 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:41.916 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:10:41 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:10:41] "GET /metrics HTTP/1.1" 200 207949 "" "Prometheus/2.33.4" 2024-09-13T23:10:42.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:41 smithi022 ceph-mon[23758]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:42.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:41 smithi022 ceph-mon[28440]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:42.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:41 smithi076 ceph-mon[24587]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:43.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:42 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:10:43.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:42 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:10:43.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:42 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:10:43.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:42 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:10:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:42 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:10:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:42 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:10:43.921 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:10:43.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:10:43.922 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:10:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:10:43.922 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:10:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:10:44.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:43 smithi022 ceph-mon[23758]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:44.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:43 smithi022 ceph-mon[28440]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:43 smithi076 ceph-mon[24587]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:46.036 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:45 smithi076 ceph-mon[24587]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:46.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:45 smithi022 ceph-mon[23758]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:46.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:45 smithi022 ceph-mon[28440]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:10:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:10:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:10:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:47 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:10:47.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:10:47.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:10:47.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:10:47.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:47 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:10:47.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:10:47.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:10:47.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:10:47.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:47 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:10:48.204 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:10:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:10:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:10:48.504 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:48 smithi022 ceph-mon[23758]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:48.505 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:48 smithi022 ceph-mon[28440]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:48 smithi076 ceph-mon[24587]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:50.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:50 smithi076 ceph-mon[24587]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:50.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:50 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:50.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:50 smithi022 ceph-mon[23758]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:50.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:50 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:50.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:50 smithi022 ceph-mon[28440]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:50.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:50 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:10:51.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:10:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:10:51] "GET /metrics HTTP/1.1" 200 207946 "" "Prometheus/2.33.4" 2024-09-13T23:10:52.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:52 smithi076 ceph-mon[24587]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:52.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:52 smithi022 ceph-mon[23758]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:52.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:52 smithi022 ceph-mon[28440]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:53.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:10:53.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:10:53.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:10:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:10:53.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:10:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:10:53.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:10:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:54 smithi076 ceph-mon[24587]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:54.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:54 smithi022 ceph-mon[23758]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:54.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:54 smithi022 ceph-mon[28440]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:10:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:56 smithi076 ceph-mon[24587]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:56.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:56 smithi022 ceph-mon[23758]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:56.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:56 smithi022 ceph-mon[28440]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:58.219 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:10:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:10:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:10:58.513 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:10:58 smithi022 ceph-mon[23758]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:58.513 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:10:58 smithi022 ceph-mon[28440]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:10:58.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:10:58 smithi076 ceph-mon[24587]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:00 smithi076 ceph-mon[24587]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:00 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:00.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:00 smithi022 ceph-mon[28440]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:00.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:00 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:00.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:00 smithi022 ceph-mon[23758]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:00.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:00 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:01.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:11:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:11:01] "GET /metrics HTTP/1.1" 200 207939 "" "Prometheus/2.33.4" 2024-09-13T23:11:02.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:02 smithi076 ceph-mon[24587]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:02.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:02 smithi022 ceph-mon[28440]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:02.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:02 smithi022 ceph-mon[23758]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:03.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:11:03.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:11:03.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:11:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:11:03.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:11:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:11:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:04 smithi076 ceph-mon[24587]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:04.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:04 smithi022 ceph-mon[28440]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:04.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:04 smithi022 ceph-mon[23758]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:05 smithi076 ceph-mon[24587]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:05.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:05 smithi022 ceph-mon[23758]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:05.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:05 smithi022 ceph-mon[28440]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:08.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:07 smithi022 ceph-mon[23758]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:08.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:07 smithi022 ceph-mon[28440]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:07 smithi076 ceph-mon[24587]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:08.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:11:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:11:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:11:10.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:09 smithi022 ceph-mon[23758]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:10.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:09 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:10.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:09 smithi022 ceph-mon[28440]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:09 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:10.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:09 smithi076 ceph-mon[24587]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:09 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:11.861 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:11:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:11:11] "GET /metrics HTTP/1.1" 200 207939 "" "Prometheus/2.33.4" 2024-09-13T23:11:12.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:11 smithi022 ceph-mon[23758]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:12.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:11 smithi022 ceph-mon[28440]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:11 smithi076 ceph-mon[24587]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:13.863 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:11:13.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:11:13.864 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:11:13.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:11:13.864 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:11:13.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:11:14.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:13 smithi022 ceph-mon[28440]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:14.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:13 smithi022 ceph-mon[23758]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:14.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:13 smithi076 ceph-mon[24587]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:16.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:15 smithi022 ceph-mon[28440]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:16.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:15 smithi022 ceph-mon[23758]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:15 smithi076 ceph-mon[24587]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:18.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:17 smithi022 ceph-mon[28440]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:18.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:17 smithi022 ceph-mon[23758]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:17 smithi076 ceph-mon[24587]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:18.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:11:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:11:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:11:20.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:19 smithi022 ceph-mon[23758]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:20.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:19 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:19 smithi022 ceph-mon[28440]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:19 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:19 smithi076 ceph-mon[24587]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:19 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:21.876 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:11:21 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:11:21] "GET /metrics HTTP/1.1" 200 207948 "" "Prometheus/2.33.4" 2024-09-13T23:11:22.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:21 smithi022 ceph-mon[23758]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:22.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:21 smithi022 ceph-mon[28440]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:22.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:21 smithi076 ceph-mon[24587]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:23.879 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:23 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:11:23.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:11:23.879 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:23 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:11:23.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:11:23.879 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:23 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:11:23.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:11:24.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:23 smithi022 ceph-mon[23758]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:24.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:23 smithi022 ceph-mon[28440]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:24.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:23 smithi076 ceph-mon[24587]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:26.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:25 smithi022 ceph-mon[23758]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:26.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:25 smithi022 ceph-mon[28440]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:25 smithi076 ceph-mon[24587]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:28.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:27 smithi022 ceph-mon[23758]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:28.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:27 smithi022 ceph-mon[28440]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:27 smithi076 ceph-mon[24587]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:28.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:11:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:11:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:11:30.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:29 smithi022 ceph-mon[23758]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:30.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:29 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:30.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:29 smithi022 ceph-mon[28440]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:30.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:29 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:29 smithi076 ceph-mon[24587]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:29 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:31.890 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:11:31 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:11:31] "GET /metrics HTTP/1.1" 200 207946 "" "Prometheus/2.33.4" 2024-09-13T23:11:32.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:31 smithi022 ceph-mon[23758]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:32.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:31 smithi022 ceph-mon[28440]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:32.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:31 smithi076 ceph-mon[24587]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:33.893 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:11:33.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:11:33.893 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:11:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:11:33.893 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:11:33.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:11:34.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:33 smithi022 ceph-mon[28440]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:34.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:33 smithi022 ceph-mon[23758]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:33 smithi076 ceph-mon[24587]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:36.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:35 smithi022 ceph-mon[28440]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:36.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:35 smithi022 ceph-mon[23758]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:36.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:35 smithi076 ceph-mon[24587]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:37 smithi022 ceph-mon[23758]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:38.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:37 smithi022 ceph-mon[28440]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:37 smithi076 ceph-mon[24587]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:38.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:11:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:11:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:11:40.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:39 smithi022 ceph-mon[28440]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:40.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:39 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:39 smithi022 ceph-mon[23758]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:40.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:39 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:39 smithi076 ceph-mon[24587]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:39 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:41.904 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:11:41 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:11:41] "GET /metrics HTTP/1.1" 200 207946 "" "Prometheus/2.33.4" 2024-09-13T23:11:42.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:41 smithi022 ceph-mon[23758]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:42.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:41 smithi022 ceph-mon[28440]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:41 smithi076 ceph-mon[24587]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:43.907 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:11:43.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:11:43.907 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:11:43.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:11:43.907 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:11:43.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:11:44.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:43 smithi022 ceph-mon[28440]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:44.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:43 smithi022 ceph-mon[23758]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:43 smithi076 ceph-mon[24587]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:46.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:45 smithi022 ceph-mon[28440]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:46.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:45 smithi022 ceph-mon[23758]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:46.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:45 smithi076 ceph-mon[24587]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:47.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:11:47.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:11:47.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:11:47.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:11:47.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:46 smithi022 ceph-mon[28440]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:11:47.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:11:47.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:11:47.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:11:47.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:11:47.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:46 smithi022 ceph-mon[23758]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:11:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:11:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:11:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:11:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:11:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:46 smithi076 ceph-mon[24587]: from='mgr.24352 172.21.15.22:0/349872050' entity='mgr.y' 2024-09-13T23:11:48.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:47 smithi022 ceph-mon[23758]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:48.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:47 smithi022 ceph-mon[28440]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:48.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:47 smithi076 ceph-mon[24587]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:48.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:11:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:11:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:11:50.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:49 smithi022 ceph-mon[23758]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:50.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:49 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:50.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:49 smithi022 ceph-mon[28440]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:50.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:49 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:49 smithi076 ceph-mon[24587]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:49 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:11:51.925 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:11:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:11:51] "GET /metrics HTTP/1.1" 200 207941 "" "Prometheus/2.33.4" 2024-09-13T23:11:52.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:51 smithi022 ceph-mon[23758]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:52.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:51 smithi022 ceph-mon[28440]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:51 smithi076 ceph-mon[24587]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:53.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:11:53.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:11:53.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:11:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:11:53.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:11:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:11:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:11:54.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:53 smithi076 ceph-mon[24587]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:54.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:53 smithi022 ceph-mon[23758]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:54.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:53 smithi022 ceph-mon[28440]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:11:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:55 smithi076 ceph-mon[24587]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:56.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:55 smithi022 ceph-mon[23758]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:55 smithi022 ceph-mon[28440]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:57 smithi076 ceph-mon[24587]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:58.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:11:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:11:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:11:58.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:57 smithi022 ceph-mon[23758]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:11:58.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:57 smithi022 ceph-mon[28440]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:00.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:59 smithi076 ceph-mon[24587]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:11:59 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:00.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:59 smithi022 ceph-mon[28440]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:00.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:11:59 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:00.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:59 smithi022 ceph-mon[23758]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:00.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:11:59 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:01.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:12:01] "GET /metrics HTTP/1.1" 200 207942 "" "Prometheus/2.33.4" 2024-09-13T23:12:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:01 smithi076 ceph-mon[24587]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:02.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:01 smithi022 ceph-mon[23758]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:02.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:01 smithi022 ceph-mon[28440]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:03.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:12:03.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:12:03.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:12:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:12:03.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:12:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:12:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:03 smithi076 ceph-mon[24587]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:04.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:03 smithi022 ceph-mon[23758]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:04.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:03 smithi022 ceph-mon[28440]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:05 smithi076 ceph-mon[24587]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:06.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:05 smithi022 ceph-mon[23758]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:06.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:05 smithi022 ceph-mon[28440]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:08.228 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:07 smithi022 ceph-mon[23758]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:08.228 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:07 smithi022 ceph-mon[28440]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:07 smithi076 ceph-mon[24587]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:08.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:12:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:12:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:09 smithi076 ceph-mon[24587]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:09 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:10.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:09 smithi022 ceph-mon[23758]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:10.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:09 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:10.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:09 smithi022 ceph-mon[28440]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:10.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:09 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:11.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:12:11] "GET /metrics HTTP/1.1" 200 207942 "" "Prometheus/2.33.4" 2024-09-13T23:12:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:11 smithi076 ceph-mon[24587]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:12.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:11 smithi022 ceph-mon[23758]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:12.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:11 smithi022 ceph-mon[28440]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:13.676 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:12:13.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:12:13.676 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:12:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:12:13.676 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:12:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:12:13.920 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:12:14.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:13 smithi022 ceph-mon[23758]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:14.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:13 smithi022 ceph-mon[28440]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:14.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:13 smithi076 ceph-mon[24587]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:16.079 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:12:16.079 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (6m) 2m ago 7m 24.4M - ba2b418f427c 863a54ec4b87 2024-09-13T23:12:16.079 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (7m) 89s ago 7m 51.2M - 8.3.5 dad864ee21e9 c818389ee4e7 2024-09-13T23:12:16.079 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (6m) 2m ago 6m 49.0M - 3.5 e1d6a67b021e 0dca3d2f62a7 2024-09-13T23:12:16.079 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283 running (2m) 89s ago 12m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 194c1a9a8cc9 2024-09-13T23:12:16.079 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:9283 running (13m) 2m ago 13m 459M - 17.2.0 e1d6a67b021e 07f20d2b59b7 2024-09-13T23:12:16.080 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (13m) 2m ago 13m 49.1M 2048M 17.2.0 e1d6a67b021e 7d59bd9c71e8 2024-09-13T23:12:16.080 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (12m) 89s ago 12m 41.9M 2048M 17.2.0 e1d6a67b021e a1194b4b1c43 2024-09-13T23:12:16.080 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (12m) 2m ago 12m 39.5M 2048M 17.2.0 e1d6a67b021e 336a6f00a2a7 2024-09-13T23:12:16.080 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (8m) 2m ago 8m 16.0M - 1dbe0e931976 68ffcaba64d7 2024-09-13T23:12:16.080 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (8m) 89s ago 8m 17.6M - 1dbe0e931976 6ee1b8ce1803 2024-09-13T23:12:16.080 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (11m) 2m ago 11m 49.1M 4017M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:12:16.080 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (11m) 2m ago 11m 50.5M 4017M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:12:16.080 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (10m) 2m ago 10m 46.1M 4017M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:12:16.080 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (10m) 2m ago 10m 48.0M 4017M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:12:16.080 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (10m) 89s ago 10m 50.0M 4272M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:12:16.080 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (9m) 89s ago 9m 51.5M 4272M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:12:16.080 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (9m) 89s ago 9m 49.4M 4272M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:12:16.081 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (8m) 89s ago 8m 49.4M 4272M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:12:16.081 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (6m) 89s ago 8m 66.9M - 514e6a882f6e 8e9cd8b611e3 2024-09-13T23:12:16.081 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (6m) 2m ago 6m 71.2M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:12:16.081 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (6m) 89s ago 6m 69.8M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:12:16.090 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:15 smithi022 ceph-mon[23758]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:16.091 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:15 smithi022 ceph-mon[28440]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:15 smithi076 ceph-mon[24587]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:16.722 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T23:12:17.242 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:16 smithi022 ceph-mon[28440]: from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:12:17.242 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:16 smithi022 ceph-mon[23758]: from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:12:17.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:16 smithi076 ceph-mon[24587]: from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:12:18.215 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:17 smithi022 ceph-mon[28440]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:18.216 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:17 smithi022 ceph-mon[23758]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:17 smithi076 ceph-mon[24587]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:18.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:12:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:12:18.976 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:12:18.976 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:12:18.976 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T23:12:18.976 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:12:18.976 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:12:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T23:12:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T23:12:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:12:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:12:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T23:12:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:12:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": {}, 2024-09-13T23:12:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:12:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:12:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:12:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:12:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-09-13T23:12:18.978 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T23:12:18.978 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:12:18.978 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:12:19.723 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph -s' 2024-09-13T23:12:20.243 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:19 smithi022 ceph-mon[23758]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:20.243 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:19 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:20.243 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:19 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2871216786' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:12:20.243 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:19 smithi022 ceph-mon[28440]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:20.243 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:19 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:20.244 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:19 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2871216786' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:12:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:19 smithi076 ceph-mon[24587]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:19 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:19 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2871216786' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:12:21.729 INFO:teuthology.orchestra.run.smithi022.stdout: cluster: 2024-09-13T23:12:21.729 INFO:teuthology.orchestra.run.smithi022.stdout: id: 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:12:21.729 INFO:teuthology.orchestra.run.smithi022.stdout: health: HEALTH_OK 2024-09-13T23:12:21.729 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:12:21.729 INFO:teuthology.orchestra.run.smithi022.stdout: services: 2024-09-13T23:12:21.729 INFO:teuthology.orchestra.run.smithi022.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-09-13T23:12:21.729 INFO:teuthology.orchestra.run.smithi022.stdout: mgr: y(active, since 8m), standbys: x 2024-09-13T23:12:21.729 INFO:teuthology.orchestra.run.smithi022.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-09-13T23:12:21.729 INFO:teuthology.orchestra.run.smithi022.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-13T23:12:21.730 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:12:21.730 INFO:teuthology.orchestra.run.smithi022.stdout: data: 2024-09-13T23:12:21.730 INFO:teuthology.orchestra.run.smithi022.stdout: pools: 6 pools, 161 pgs 2024-09-13T23:12:21.730 INFO:teuthology.orchestra.run.smithi022.stdout: objects: 209 objects, 585 KiB 2024-09-13T23:12:21.730 INFO:teuthology.orchestra.run.smithi022.stdout: usage: 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:21.730 INFO:teuthology.orchestra.run.smithi022.stdout: pgs: 161 active+clean 2024-09-13T23:12:21.730 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:12:21.730 INFO:teuthology.orchestra.run.smithi022.stdout: io: 2024-09-13T23:12:21.730 INFO:teuthology.orchestra.run.smithi022.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-13T23:12:21.730 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:12:21.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:21 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:12:21] "GET /metrics HTTP/1.1" 200 207949 "" "Prometheus/2.33.4" 2024-09-13T23:12:22.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:21 smithi076 ceph-mon[24587]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:22.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:21 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/708614284' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T23:12:22.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:21 smithi022 ceph-mon[23758]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:22.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:21 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/708614284' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T23:12:22.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:21 smithi022 ceph-mon[28440]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:22.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:21 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/708614284' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T23:12:22.334 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T23:12:23.815 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:23 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:12:23.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:12:23.816 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:23 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:12:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:12:23.816 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:23 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:12:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:12:24.121 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:23 smithi022 ceph-mon[28440]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:24.121 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:23 smithi022 ceph-mon[23758]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:24.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:23 smithi076 ceph-mon[24587]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:24.494 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-09-13T23:12:25.098 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:24 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/520165752' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:12:25.098 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:24 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/520165752' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:12:25.124 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-09-13T23:12:25.279 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:24 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/520165752' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:12:26.112 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:25 smithi022 ceph-mon[23758]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:26.113 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:25 smithi022 ceph-mon[28440]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:25 smithi076 ceph-mon[24587]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:27.239 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:12:27.945 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph mgr fail' 2024-09-13T23:12:28.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:27 smithi022 ceph-mon[23758]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:28.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:27 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3116443111' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:12:28.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:27 smithi022 ceph-mon[28440]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:28.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:27 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3116443111' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:12:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:27 smithi076 ceph-mon[24587]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:27 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3116443111' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:12:28.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:12:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:12:30.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:29 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:12:29] ENGINE Bus STOPPING 2024-09-13T23:12:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:29 smithi076 ceph-mon[24587]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:29 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:29 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3355690022' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:12:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:29 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:12:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:29 smithi076 ceph-mon[24587]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T23:12:30.352 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:29.983+0000 7f2fe2dfc700 -1 mgr handle_mgr_map I was active but no longer amignoring --setuser ceph since I am not root 2024-09-13T23:12:30.352 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ignoring --setgroup ceph since I am not root 2024-09-13T23:12:30.352 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mgr[23990]: -- 172.21.15.22:0/887850549 <== mon.1 v2:172.21.15.22:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560cdbf82340 con 0x560cdbf5ac00 2024-09-13T23:12:30.352 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:30.155+0000 7fe1d6391000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T23:12:30.352 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:30.237+0000 7fe1d6391000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T23:12:30.353 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:29 smithi022 ceph-mon[23758]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:30.353 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:29 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:30.353 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:29 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3355690022' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:12:30.353 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:29 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:12:30.353 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:29 smithi022 ceph-mon[23758]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T23:12:30.353 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:29 smithi022 ceph-mon[28440]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:30.353 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:29 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:30.353 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:29 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3355690022' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:12:30.354 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:29 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:12:30.354 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:29 smithi022 ceph-mon[28440]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T23:12:30.600 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:12:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T23:12:30.600 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:12:30] ENGINE Bus STOPPED 2024-09-13T23:12:30.600 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:12:30] ENGINE Bus STARTING 2024-09-13T23:12:30.600 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:12:30] ENGINE Serving on http://:::9283 2024-09-13T23:12:30.600 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:12:30] ENGINE Bus STARTED 2024-09-13T23:12:30.651 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:30.434+0000 7fe1d6391000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T23:12:30.652 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:30.528+0000 7fe1d6391000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T23:12:30.677 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 180' 2024-09-13T23:12:30.907 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:30.668+0000 7fe1d6391000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:12:31.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:30.906+0000 7fe1d6391000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T23:12:31.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T23:12:31.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: mgrmap e23: x(active, starting, since 0.0785689s) 2024-09-13T23:12:31.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:12:31.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:12:31.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:12:31.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:12:31.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:12:31.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:12:31.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: Manager daemon x is now available 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: Queued rgw.foo for migration 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:31.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: Migrating certs/keys for iscsi.foo spec to cert store 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: Migrating certs/keys for rgw.foo spec to cert store 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: Checking for cert/key for grafana.a 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: mgrmap e23: x(active, starting, since 0.0785689s) 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:12:31.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: Manager daemon x is now available 2024-09-13T23:12:31.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:31.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: Queued rgw.foo for migration 2024-09-13T23:12:31.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-13T23:12:31.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:31.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: Migrating certs/keys for iscsi.foo spec to cert store 2024-09-13T23:12:31.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: Migrating certs/keys for rgw.foo spec to cert store 2024-09-13T23:12:31.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: Checking for cert/key for grafana.a 2024-09-13T23:12:31.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:31.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:12:31.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:12:31.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:12:31.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T23:12:31.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:30 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T23:12:31.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T23:12:31.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: mgrmap e23: x(active, starting, since 0.0785689s) 2024-09-13T23:12:31.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:12:31.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:12:31.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:12:31.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:12:31.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:12:31.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:12:31.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:12:31.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:12:31.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:12:31.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:12:31.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: Manager daemon x is now available 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: Queued rgw.foo for migration 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: Migrating certs/keys for iscsi.foo spec to cert store 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: Migrating certs/keys for rgw.foo spec to cert store 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: Checking for cert/key for grafana.a 2024-09-13T23:12:31.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:31.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:12:31.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:12:31.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:12:31.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T23:12:31.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:30 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T23:12:31.536 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:31 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:31.268+0000 7fe1d6391000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T23:12:31.536 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:31 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:31.363+0000 7fe1d6391000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T23:12:31.536 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:31 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:31.534+0000 7fe1d6391000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T23:12:32.273 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-mon[23758]: Deploying cephadm binary to smithi076 2024-09-13T23:12:32.273 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-mon[23758]: Deploying cephadm binary to smithi022 2024-09-13T23:12:32.273 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-mon[23758]: mgrmap e24: x(active, since 1.09549s) 2024-09-13T23:12:32.273 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-mon[23758]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:32.274 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:32.274 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:32.274 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-mon[28440]: Deploying cephadm binary to smithi076 2024-09-13T23:12:32.274 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-mon[28440]: Deploying cephadm binary to smithi022 2024-09-13T23:12:32.274 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-mon[28440]: mgrmap e24: x(active, since 1.09549s) 2024-09-13T23:12:32.274 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-mon[28440]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:32.274 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:32.274 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:32.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-mon[24587]: Deploying cephadm binary to smithi076 2024-09-13T23:12:32.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-mon[24587]: Deploying cephadm binary to smithi022 2024-09-13T23:12:32.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-mon[24587]: mgrmap e24: x(active, since 1.09549s) 2024-09-13T23:12:32.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-mon[24587]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:32.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:32.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:32.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:12:32.168+0000 7f400ae1e640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:23:12:32] ENGINE Error in HTTPServer.serve 2024-09-13T23:12:32.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:12:32.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:12:32.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: self._connections.run(self.expiration_interval) 2024-09-13T23:12:32.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:12:32.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: self._run(expiration_interval) 2024-09-13T23:12:32.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:12:32.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:12:32.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:12:32.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:12:32.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:12:32.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: s = self.context.wrap_socket( 2024-09-13T23:12:32.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:12:32.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.sslsocket_class._create( 2024-09-13T23:12:32.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:12:32.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: self.do_handshake() 2024-09-13T23:12:32.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:12:32.289 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: self._sslobj.do_handshake() 2024-09-13T23:12:32.289 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:12:32.289 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:12:32.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:32.666+0000 7fe1d6391000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T23:12:32.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:32.770+0000 7fe1d6391000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T23:12:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:33 smithi076 ceph-mon[24587]: [13/Sep/2024:23:12:31] ENGINE Bus STARTING 2024-09-13T23:12:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:33 smithi076 ceph-mon[24587]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:33 smithi076 ceph-mon[24587]: [13/Sep/2024:23:12:32] ENGINE Serving on https://172.21.15.76:7150 2024-09-13T23:12:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:33 smithi076 ceph-mon[24587]: [13/Sep/2024:23:12:32] ENGINE Error in HTTPServer.serve 2024-09-13T23:12:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._run(expiration_interval) 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: s = self.context.wrap_socket( 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.sslsocket_class._create( 2024-09-13T23:12:33.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:12:33.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: self.do_handshake() 2024-09-13T23:12:33.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:12:33.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._sslobj.do_handshake() 2024-09-13T23:12:33.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:12:33.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:33 smithi076 ceph-mon[24587]: [13/Sep/2024:23:12:32] ENGINE Serving on http://172.21.15.76:8765 2024-09-13T23:12:33.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:33 smithi076 ceph-mon[24587]: [13/Sep/2024:23:12:32] ENGINE Bus STARTED 2024-09-13T23:12:33.332 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-mon[28440]: [13/Sep/2024:23:12:31] ENGINE Bus STARTING 2024-09-13T23:12:33.332 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-mon[28440]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:33.332 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-mon[28440]: [13/Sep/2024:23:12:32] ENGINE Serving on https://172.21.15.76:7150 2024-09-13T23:12:33.332 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-mon[28440]: [13/Sep/2024:23:12:32] ENGINE Error in HTTPServer.serve 2024-09-13T23:12:33.333 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:12:33.333 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:12:33.333 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:12:33.333 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:12:33.333 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._run(expiration_interval) 2024-09-13T23:12:33.333 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:12:33.333 INFO:journalctl@ceph.mon.c.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:12:33.333 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:12:33.333 INFO:journalctl@ceph.mon.c.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:12:33.333 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:12:33.335 INFO:journalctl@ceph.mon.c.smithi022.stdout: s = self.context.wrap_socket( 2024-09-13T23:12:33.335 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:12:33.336 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.sslsocket_class._create( 2024-09-13T23:12:33.336 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:12:33.336 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.do_handshake() 2024-09-13T23:12:33.336 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:12:33.336 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._sslobj.do_handshake() 2024-09-13T23:12:33.336 INFO:journalctl@ceph.mon.c.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:12:33.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-mon[28440]: [13/Sep/2024:23:12:32] ENGINE Serving on http://172.21.15.76:8765 2024-09-13T23:12:33.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-mon[28440]: [13/Sep/2024:23:12:32] ENGINE Bus STARTED 2024-09-13T23:12:33.336 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-mon[23758]: [13/Sep/2024:23:12:31] ENGINE Bus STARTING 2024-09-13T23:12:33.336 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-mon[23758]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-mon[23758]: [13/Sep/2024:23:12:32] ENGINE Serving on https://172.21.15.76:7150 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-mon[23758]: [13/Sep/2024:23:12:32] ENGINE Error in HTTPServer.serve 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._run(expiration_interval) 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:12:33.337 INFO:journalctl@ceph.mon.a.smithi022.stdout: s = self.context.wrap_socket( 2024-09-13T23:12:33.338 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:12:33.338 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.sslsocket_class._create( 2024-09-13T23:12:33.338 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:12:33.338 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.do_handshake() 2024-09-13T23:12:33.338 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:12:33.338 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._sslobj.do_handshake() 2024-09-13T23:12:33.338 INFO:journalctl@ceph.mon.a.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:12:33.338 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-mon[23758]: [13/Sep/2024:23:12:32] ENGINE Serving on http://172.21.15.76:8765 2024-09-13T23:12:33.338 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-mon[23758]: [13/Sep/2024:23:12:32] ENGINE Bus STARTED 2024-09-13T23:12:33.676 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:12:33.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:12:33.676 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:12:33.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": dial tcp 172.21.15.22:8443: connect: connection refused" 2024-09-13T23:12:33.676 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:12:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:12:34.013 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:33.762+0000 7fe1d6391000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T23:12:34.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:34 smithi076 ceph-mon[24587]: mgrmap e25: x(active, since 3s) 2024-09-13T23:12:34.337 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:34 smithi022 ceph-mon[28440]: mgrmap e25: x(active, since 3s) 2024-09-13T23:12:34.337 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:34 smithi022 ceph-mon[23758]: mgrmap e25: x(active, since 3s) 2024-09-13T23:12:34.338 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:34.097+0000 7fe1d6391000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T23:12:34.338 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:34.241+0000 7fe1d6391000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:12:34.338 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:34.336+0000 7fe1d6391000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T23:12:34.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:34.440+0000 7fe1d6391000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T23:12:34.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:34.534+0000 7fe1d6391000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T23:12:35.014 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:34.702+0000 7fe1d6391000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T23:12:35.014 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:34.801+0000 7fe1d6391000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T23:12:35.014 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:34.906+0000 7fe1d6391000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T23:12:35.261 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:35 smithi076 ceph-mon[24587]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:35.302 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:35 smithi022 ceph-mon[23758]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:35.302 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:35 smithi022 ceph-mon[28440]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:35.303 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:35.206+0000 7fe1d6391000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T23:12:35.303 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:35.301+0000 7fe1d6391000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T23:12:35.660 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:35.410+0000 7fe1d6391000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T23:12:36.675 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:36 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:36.562+0000 7fe1d6391000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T23:12:36.675 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:36 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:23:12:36] ENGINE Bus STARTING 2024-09-13T23:12:36.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:36 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: CherryPy Checker: 2024-09-13T23:12:36.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:36 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: The Application mounted at '' has an empty config. 2024-09-13T23:12:36.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:36 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: 2024-09-13T23:12:36.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:36 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:23:12:36] ENGINE Serving on http://:::9283 2024-09-13T23:12:36.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:36 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: [13/Sep/2024:23:12:36] ENGINE Bus STARTED 2024-09-13T23:12:37.826 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:37.826 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: Standby manager daemon y started 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.22:0/1201962977' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.22:0/1201962977' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.22:0/1201962977' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.22:0/1201962977' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.827 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.828 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.828 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:37.828 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: Standby manager daemon y started 2024-09-13T23:12:37.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.22:0/1201962977' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T23:12:37.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.22:0/1201962977' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:12:37.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.22:0/1201962977' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T23:12:37.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.22:0/1201962977' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:12:37.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.830 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.830 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.830 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.830 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.830 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.830 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:37.830 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.830 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:37.830 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:12:37] "GET /metrics HTTP/1.1" 200 34890 "" "Prometheus/2.33.4" 2024-09-13T23:12:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:12:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: Standby manager daemon y started 2024-09-13T23:12:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.22:0/1201962977' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T23:12:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.22:0/1201962977' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:12:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.22:0/1201962977' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T23:12:38.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.22:0/1201962977' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:12:38.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:37 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.176 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:37 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:12:37.825Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=5 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:12:38.915 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: Adjusting osd_memory_target on smithi076 to 3248M 2024-09-13T23:12:38.915 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.915 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.915 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.915 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.915 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: mgrmap e26: x(active, since 7s), standbys: y 2024-09-13T23:12:38.915 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:12:38.915 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.915 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: Adjusting osd_memory_target on smithi022 to 2993M 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.917 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: Adjusting osd_memory_target on smithi076 to 3248M 2024-09-13T23:12:38.917 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.917 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.917 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.917 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.917 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: mgrmap e26: x(active, since 7s), standbys: y 2024-09-13T23:12:38.917 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:12:38.917 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.917 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.917 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.918 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:38.918 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: Adjusting osd_memory_target on smithi022 to 2993M 2024-09-13T23:12:38.918 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.918 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:12:38.918 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:12:38.918 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:12:38.918 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:12:38.918 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.918 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.918 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.918 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:38.918 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: Adjusting osd_memory_target on smithi076 to 3248M 2024-09-13T23:12:39.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:39.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:39.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:39.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:39.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: mgrmap e26: x(active, since 7s), standbys: y 2024-09-13T23:12:39.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:12:39.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: Adjusting osd_memory_target on smithi022 to 2993M 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.176 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:39 smithi022 systemd[1]: Stopping Ceph node-exporter.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:12:39.676 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:39 smithi022 podman[57402]: 2024-09-13 23:12:39.389659286 +0000 UTC m=+0.116449708 container died 68ffcaba64d77df95582b553f6fa61fea9d69ee25e29a0c5ef3d5cf04a8aee22 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T23:12:39.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:12:39.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:12:39.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T23:12:39.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:12:39.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:12:39.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:12:39.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:12:39.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T23:12:39.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: Deploying daemon node-exporter.a on smithi022 2024-09-13T23:12:39.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:39.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:12:39.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:12:39.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T23:12:39.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:12:39.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:12:39.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:12:39.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:12:39.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:39.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T23:12:39.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: Deploying daemon node-exporter.a on smithi022 2024-09-13T23:12:39.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:39 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:40.012 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:39 smithi022 podman[57402]: 2024-09-13 23:12:39.794913812 +0000 UTC m=+0.521704235 container cleanup 68ffcaba64d77df95582b553f6fa61fea9d69ee25e29a0c5ef3d5cf04a8aee22 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T23:12:40.013 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:39 smithi022 bash[57402]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T23:12:40.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:12:40.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:12:40.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T23:12:40.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:12:40.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:12:40.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:12:40.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:12:40.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:40.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:40.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:40.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:40.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T23:12:40.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: Deploying daemon node-exporter.a on smithi022 2024-09-13T23:12:40.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:39 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:40.377 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:40 smithi022 podman[57414]: 2024-09-13 23:12:40.012597613 +0000 UTC m=+0.622089132 container remove 68ffcaba64d77df95582b553f6fa61fea9d69ee25e29a0c5ef3d5cf04a8aee22 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T23:12:40.378 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:40 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T23:12:40.378 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:40 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T23:12:40.378 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:40 smithi022 systemd[1]: Stopped Ceph node-exporter.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:12:40.378 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:40 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 4.513s CPU time. 2024-09-13T23:12:40.676 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:40 smithi022 systemd[1]: Starting Ceph node-exporter.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:12:40.676 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:40 smithi022 bash[57556]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-13T23:12:41.426 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:41 smithi022 bash[57556]: Getting image source signatures 2024-09-13T23:12:41.426 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:41 smithi022 bash[57556]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-13T23:12:41.426 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:41 smithi022 bash[57556]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-13T23:12:41.426 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:41 smithi022 bash[57556]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-13T23:12:41.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:41 smithi022 ceph-mon[23758]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T23:12:41.928 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:41 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:12:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:12:41.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:41 smithi022 ceph-mon[28440]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T23:12:42.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:41 smithi076 ceph-mon[24587]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T23:12:42.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:42 smithi022 bash[57556]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-13T23:12:42.879 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:42 smithi022 bash[57556]: Writing manifest to image destination 2024-09-13T23:12:43.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:42 smithi022 ceph-mon[23758]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T23:12:43.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:42 smithi022 ceph-mon[28440]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T23:12:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:42 smithi076 ceph-mon[24587]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T23:12:43.519 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 podman[57556]: 2024-09-13 23:12:43.126492017 +0000 UTC m=+2.650862283 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T23:12:43.519 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 podman[57556]: 2024-09-13 23:12:43.224670894 +0000 UTC m=+2.749041158 container create 892ef92b2fceda848c9cd944e523f6a66e58e6d5da786f1a08fc573c8315b007 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T23:12:43.520 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 podman[57556]: 2024-09-13 23:12:43.434932663 +0000 UTC m=+2.959302931 container init 892ef92b2fceda848c9cd944e523f6a66e58e6d5da786f1a08fc573c8315b007 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T23:12:43.520 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 podman[57556]: 2024-09-13 23:12:43.439675445 +0000 UTC m=+2.964045711 container start 892ef92b2fceda848c9cd944e523f6a66e58e6d5da786f1a08fc573c8315b007 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T23:12:43.520 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.444Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T23:12:43.520 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.444Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T23:12:43.520 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.444Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T23:12:43.520 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.444Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T23:12:43.520 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.444Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T23:12:43.520 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.444Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T23:12:43.520 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T23:12:43.520 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T23:12:43.520 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T23:12:43.521 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T23:12:43.521 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T23:12:43.521 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T23:12:43.521 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T23:12:43.521 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T23:12:43.521 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T23:12:43.522 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T23:12:43.523 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T23:12:43.523 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T23:12:43.523 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T23:12:43.523 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T23:12:43.523 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T23:12:43.523 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T23:12:43.523 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T23:12:43.523 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T23:12:43.523 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T23:12:43.523 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T23:12:43.523 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T23:12:43.523 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T23:12:43.524 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a[57631]: ts=2024-09-13T23:12:43.445Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T23:12:43.525 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 bash[57556]: 892ef92b2fceda848c9cd944e523f6a66e58e6d5da786f1a08fc573c8315b007 2024-09-13T23:12:43.926 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:12:43 smithi022 systemd[1]: Started Ceph node-exporter.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:12:43.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=error ts=2024-09-13T23:12:43.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:12:43.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:12:43.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.22:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.22 because it doesn't contain any IP SANs" 2024-09-13T23:12:43.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:43 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=warn ts=2024-09-13T23:12:43.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.76:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.76 because it doesn't contain any IP SANs" 2024-09-13T23:12:44.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:44 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:44.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:44 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:44.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:44 smithi022 ceph-mon[23758]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T23:12:44.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:44 smithi022 ceph-mon[23758]: Deploying daemon alertmanager.a on smithi022 2024-09-13T23:12:44.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:44 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:44.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:44 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:44.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:44 smithi022 ceph-mon[28440]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T23:12:44.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:44 smithi022 ceph-mon[28440]: Deploying daemon alertmanager.a on smithi022 2024-09-13T23:12:45.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:44 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:45.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:44 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:45.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:44 smithi076 ceph-mon[24587]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T23:12:45.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:44 smithi076 ceph-mon[24587]: Deploying daemon alertmanager.a on smithi022 2024-09-13T23:12:45.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:45 smithi022 ceph-mon[23758]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:12:45.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:45 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:12:45.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:45 smithi022 ceph-mon[28440]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:12:45.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:45 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:12:46.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:45 smithi076 ceph-mon[24587]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:12:46.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:45 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:12:47.900 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:47 smithi022 ceph-mon[23758]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:12:47.901 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:47 smithi022 ceph-mon[28440]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:12:48.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:47 smithi076 ceph-mon[24587]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:12:48.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:12:47] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-09-13T23:12:49.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:49 smithi022 ceph-mon[28440]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:12:49.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:49 smithi022 ceph-mon[28440]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:49.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:49 smithi022 ceph-mon[23758]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:12:49.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:49 smithi022 ceph-mon[23758]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:50.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:49 smithi076 ceph-mon[24587]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:12:50.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:49 smithi076 ceph-mon[24587]: from='client.14829 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:12:51.085 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:50 smithi022 systemd[1]: Stopping Ceph alertmanager.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:12:51.410 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[48444]: level=info ts=2024-09-13T23:12:51.085Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-09-13T23:12:51.410 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:51 smithi022 podman[57974]: 2024-09-13 23:12:51.105620289 +0000 UTC m=+0.141448563 container died 863a54ec4b873e6d13ae655068f9ec3b93afb42d53f22a93493d99aece579368 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:12:51.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:51 smithi022 ceph-mon[23758]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:51.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:12:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:12:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:12:51.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:51 smithi022 ceph-mon[28440]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:51.677 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:51 smithi022 podman[57974]: 2024-09-13 23:12:51.509742837 +0000 UTC m=+0.545571110 container cleanup 863a54ec4b873e6d13ae655068f9ec3b93afb42d53f22a93493d99aece579368 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:12:51.677 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:51 smithi022 bash[57974]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T23:12:52.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:51 smithi076 ceph-mon[24587]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:52.078 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:51 smithi022 podman[57987]: 2024-09-13 23:12:51.75257095 +0000 UTC m=+0.642168785 container remove 863a54ec4b873e6d13ae655068f9ec3b93afb42d53f22a93493d99aece579368 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:12:52.078 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:51 smithi022 podman[57987]: 2024-09-13 23:12:51.946577827 +0000 UTC m=+0.836175660 volume remove feabe4fe524a63762d0ecb1c6084ec4e2ccf78229de6803fae2aed1af3b4dbfb 2024-09-13T23:12:52.426 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:52 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T23:12:52.426 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:52 smithi022 systemd[1]: Stopped Ceph alertmanager.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:12:52.426 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:52 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.451s CPU time. 2024-09-13T23:12:52.426 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:52 smithi022 systemd[1]: Starting Ceph alertmanager.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:12:52.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:12:52 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:12:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.76:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.22:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:12:52.867 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:52 smithi022 podman[58114]: 2024-09-13 23:12:52.414759378 +0000 UTC m=+0.016630654 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-13T23:12:52.867 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:52 smithi022 podman[58114]: 2024-09-13 23:12:52.531639547 +0000 UTC m=+0.133510810 volume create d1d2a0487f276dc966eb46377801e2697c882b31ae4255d77953dc4d8665e9c3 2024-09-13T23:12:52.867 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:52 smithi022 podman[58114]: 2024-09-13 23:12:52.632039932 +0000 UTC m=+0.233911192 container create dbd41932f4749ad84480de7cfc5e763c449d3030b8fa80cad55fd8cbd1b1ce91 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:12:53.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:52 smithi022 ceph-mon[23758]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:52 smithi022 ceph-mon[28440]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:53.177 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:52 smithi022 podman[58114]: 2024-09-13 23:12:52.985184389 +0000 UTC m=+0.587055652 container init dbd41932f4749ad84480de7cfc5e763c449d3030b8fa80cad55fd8cbd1b1ce91 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:12:53.177 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:52 smithi022 podman[58114]: 2024-09-13 23:12:52.988436147 +0000 UTC m=+0.590307410 container start dbd41932f4749ad84480de7cfc5e763c449d3030b8fa80cad55fd8cbd1b1ce91 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:12:53.177 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[58129]: ts=2024-09-13T23:12:53.002Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T23:12:53.177 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[58129]: ts=2024-09-13T23:12:53.002Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T23:12:53.178 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[58129]: ts=2024-09-13T23:12:53.003Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.22 port=9094 2024-09-13T23:12:53.178 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[58129]: ts=2024-09-13T23:12:53.004Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T23:12:53.178 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[58129]: ts=2024-09-13T23:12:53.028Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T23:12:53.178 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[58129]: ts=2024-09-13T23:12:53.028Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T23:12:53.178 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[58129]: ts=2024-09-13T23:12:53.032Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T23:12:53.178 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[58129]: ts=2024-09-13T23:12:53.032Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T23:12:53.178 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:53 smithi022 bash[58114]: dbd41932f4749ad84480de7cfc5e763c449d3030b8fa80cad55fd8cbd1b1ce91 2024-09-13T23:12:53.179 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:53 smithi022 systemd[1]: Started Ceph alertmanager.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:12:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:52 smithi076 ceph-mon[24587]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:54.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:54.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:54.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[23758]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:12:54.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:12:54.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:12:54.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:12:54.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[23758]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:12:54.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:54.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:54.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[28440]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:12:54.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:12:54.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:12:54.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:12:54.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:54 smithi022 ceph-mon[28440]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:12:54.428 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:54 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:54.429 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:54 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:12:54.429 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:54 smithi076 ceph-mon[24587]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:12:54.429 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:54 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:12:54.429 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:54 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:12:54.429 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:54 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:12:54.429 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:54 smithi076 ceph-mon[24587]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:12:55.366 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:55 smithi022 ceph-mon[23758]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:55.366 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:12:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[58129]: ts=2024-09-13T23:12:55.005Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000804981s 2024-09-13T23:12:55.366 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:55 smithi022 ceph-mon[28440]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:55.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:55 smithi076 ceph-mon[24587]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:56.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:12:56.050+0000 7f403e8c2640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:12:56.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Creating ceph-iscsi config... 2024-09-13T23:12:56.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:12:56.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:12:56.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:12:56.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:56.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:56.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:12:56.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:12:56.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return _run_code(code, main_globals, None, 2024-09-13T23:12:56.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:12:56.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: exec(code, run_globals) 2024-09-13T23:12:56.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:12:56.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:12:56.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:12:56.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:12:56.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:12:56.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:12:56.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:12:56.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:56.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:56.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:12:56.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:12:56.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:12:56.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:12:56.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.wait_async( 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return future.result(timeout) 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.__get_result() 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: raise self._exception 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: out, err, code = await self._run_cephadm( 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: raise OrchestratorError( 2024-09-13T23:12:56.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Creating ceph-iscsi config... 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return _run_code(code, main_globals, None, 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: exec(code, run_globals) 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:12:56.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:12:56.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:12:56.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:12:56.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:12:56.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:12:56.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:12:56.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:56.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:57 smithi022 ceph-mon[23758]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:57 smithi022 ceph-mon[23758]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:12:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:12:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:12:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:12:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:12:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:12:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:12:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:12:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:12:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:12:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:12:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.wait_async( 2024-09-13T23:12:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:12:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:12:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:12:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout: return future.result(timeout) 2024-09-13T23:12:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:12:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.__get_result() 2024-09-13T23:12:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:12:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout: raise self._exception 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: out, err, code = await self._run_cephadm( 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: raise OrchestratorError( 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:12:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:57 smithi022 ceph-mon[23758]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:12:57.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:57 smithi022 ceph-mon[23758]: Deploying daemon prometheus.a on smithi076 2024-09-13T23:12:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:57 smithi022 ceph-mon[28440]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:57 smithi022 ceph-mon[28440]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:12:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:12:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:12:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:12:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:12:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.433 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:12:57.433 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:12:57.433 INFO:journalctl@ceph.mon.c.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:12:57.433 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:12:57.433 INFO:journalctl@ceph.mon.c.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:12:57.433 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:12:57.433 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:12:57.433 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:12:57.433 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:12:57.433 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:12:57.433 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:12:57.433 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.wait_async( 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: return future.result(timeout) 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:12:57.434 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.__get_result() 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: raise self._exception 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: out, err, code = await self._run_cephadm( 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: raise OrchestratorError( 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:12:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:57 smithi022 ceph-mon[28440]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:12:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:57 smithi022 ceph-mon[28440]: Deploying daemon prometheus.a on smithi076 2024-09-13T23:12:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:57 smithi076 ceph-mon[24587]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:12:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:57 smithi076 ceph-mon[24587]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:12:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout: Creating ceph-iscsi config... 2024-09-13T23:12:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:12:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:12:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:12:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: exec(code, run_globals) 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.wait_async( 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: return future.result(timeout) 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.__get_result() 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:12:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: raise self._exception 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: out, err, code = await self._run_cephadm( 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: raise OrchestratorError( 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: Creating ceph-iscsi config... 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:12:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: exec(code, run_globals) 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:12:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:57 smithi076 ceph-mon[24587]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:12:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:57 smithi076 ceph-mon[24587]: Deploying daemon prometheus.a on smithi076 2024-09-13T23:12:58.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:12:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:12:57] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-09-13T23:12:59.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:12:59 smithi022 ceph-mon[23758]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:59.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:12:59 smithi022 ceph-mon[28440]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:12:59.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:12:59 smithi076 ceph-mon[24587]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:13:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:01 smithi022 ceph-mon[23758]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:13:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:01 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:13:01.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:01 smithi022 ceph-mon[28440]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:13:01.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:01 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:13:01.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:01 smithi076 ceph-mon[24587]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:13:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:01 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:13:01.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:13:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[23986]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:13:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T23:13:02.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:02 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.76:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.22:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:13:03.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:02 smithi022 ceph-mon[23758]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:13:03.176 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:13:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[58129]: ts=2024-09-13T23:13:03.006Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002086181s 2024-09-13T23:13:03.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:02 smithi022 ceph-mon[28440]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:13:03.266 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:02 smithi076 ceph-mon[24587]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:13:05.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:05 smithi022 ceph-mon[23758]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:05.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:05 smithi022 ceph-mon[28440]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:05.437 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:05 smithi076 ceph-mon[24587]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:06.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:05 smithi076 systemd[1]: Stopping Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:13:06.478 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:06.186Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T23:13:06.479 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:06.186Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-09-13T23:13:06.479 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:06.186Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-09-13T23:13:06.479 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:06.186Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-09-13T23:13:06.479 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:06.186Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-09-13T23:13:06.479 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:06.186Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-09-13T23:13:06.479 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:06.186Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T23:13:06.479 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:06.186Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-09-13T23:13:06.479 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:06.186Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T23:13:06.479 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:06.186Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T23:13:06.479 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:06.186Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-09-13T23:13:06.480 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[41476]: ts=2024-09-13T23:13:06.186Z caller=main.go:1066 level=info msg="See you next time!" 2024-09-13T23:13:06.480 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 podman[47973]: 2024-09-13 23:13:06.196346967 +0000 UTC m=+0.113145168 container died 8e9cd8b611e38b6d9905423b92cebc0c50cb59cb894d7a8553eb070a54a73fab (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:13:06.786 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 podman[47973]: 2024-09-13 23:13:06.585938792 +0000 UTC m=+0.502736994 container cleanup 8e9cd8b611e38b6d9905423b92cebc0c50cb59cb894d7a8553eb070a54a73fab (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:13:06.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 bash[47973]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T23:13:07.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-mon[24587]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:07.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:06 smithi076 podman[47985]: 2024-09-13 23:13:06.962197475 +0000 UTC m=+0.767842458 container remove 8e9cd8b611e38b6d9905423b92cebc0c50cb59cb894d7a8553eb070a54a73fab (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:13:07.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T23:13:07.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 systemd[1]: Stopped Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:13:07.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 2.313s CPU time. 2024-09-13T23:13:07.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:07 smithi022 ceph-mon[23758]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:07.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:07 smithi022 ceph-mon[28440]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:07.721 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 systemd[1]: Starting Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:13:07.721 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 podman[48114]: 2024-09-13 23:13:07.472927226 +0000 UTC m=+0.017157912 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T23:13:07.721 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 podman[48114]: 2024-09-13 23:13:07.588677554 +0000 UTC m=+0.132908272 container create 3a2530c356f9b9d7ebfdc61cadd96c4bd028761c0e022e4a67df147319f9c233 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:13:08.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 podman[48114]: 2024-09-13 23:13:07.832625304 +0000 UTC m=+0.376855981 container init 3a2530c356f9b9d7ebfdc61cadd96c4bd028761c0e022e4a67df147319f9c233 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:13:08.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 podman[48114]: 2024-09-13 23:13:07.835615738 +0000 UTC m=+0.379846414 container start 3a2530c356f9b9d7ebfdc61cadd96c4bd028761c0e022e4a67df147319f9c233 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:13:08.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.859Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T23:13:08.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.860Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T23:13:08.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.860Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi076 (none))" 2024-09-13T23:13:08.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.860Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T23:13:08.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.860Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T23:13:08.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.861Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T23:13:08.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.862Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T23:13:08.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.863Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T23:13:08.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.863Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T23:13:08.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.865Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T23:13:08.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.865Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.888µs 2024-09-13T23:13:08.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.865Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T23:13:08.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.872Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-13T23:13:08.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.887Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-13T23:13:08.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.887Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-13T23:13:08.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.887Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.484µs wal_replay_duration=21.983102ms wbl_replay_duration=92ns total_replay_duration=22.021661ms 2024-09-13T23:13:08.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.889Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T23:13:08.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.889Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T23:13:08.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.889Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T23:13:08.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.911Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.850768ms db_storage=782ns remote_storage=999ns web_handler=363ns query_engine=621ns scrape=417.286µs scrape_sd=89.329µs notify=18.085µs notify_sd=15.2µs rules=20.97479ms tracing=6.177µs 2024-09-13T23:13:08.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.911Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T23:13:08.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:07.911Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T23:13:08.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 bash[48114]: 3a2530c356f9b9d7ebfdc61cadd96c4bd028761c0e022e4a67df147319f9c233 2024-09-13T23:13:08.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:07 smithi076 systemd[1]: Started Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:13:08.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:08 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/351287966' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:13:08.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:08 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/775347322' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3501087460"}]: dispatch 2024-09-13T23:13:08.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:08 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3501087460"}]: dispatch 2024-09-13T23:13:08.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:08 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:08.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:08 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:08.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:08 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/351287966' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:13:08.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:08 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/775347322' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3501087460"}]: dispatch 2024-09-13T23:13:08.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:08 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3501087460"}]: dispatch 2024-09-13T23:13:08.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:08 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:08.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:08 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:08.485 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:08 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/351287966' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:13:08.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:08 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/775347322' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3501087460"}]: dispatch 2024-09-13T23:13:08.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:08 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3501087460"}]: dispatch 2024-09-13T23:13:08.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:08 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:08.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:08 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:08.748 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:08 smithi076 systemd[1]: Stopping Ceph node-exporter.b for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:13:09.021 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:08 smithi076 podman[48294]: 2024-09-13 23:13:08.747592788 +0000 UTC m=+0.117588247 container died 6ee1b8ce1803913235dfe6676635a2582db9fd91724d8033273437f7cba5b822 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T23:13:09.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:09 smithi022 ceph-mon[28440]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T23:13:09.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:09 smithi022 ceph-mon[28440]: Deploying daemon node-exporter.b on smithi076 2024-09-13T23:13:09.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:09 smithi022 ceph-mon[28440]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:09.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:09 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3501087460"}]': finished 2024-09-13T23:13:09.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:09 smithi022 ceph-mon[28440]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T23:13:09.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:09 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2403409397' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/282633179"}]: dispatch 2024-09-13T23:13:09.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:09 smithi022 ceph-mon[23758]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T23:13:09.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:09 smithi022 ceph-mon[23758]: Deploying daemon node-exporter.b on smithi076 2024-09-13T23:13:09.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:09 smithi022 ceph-mon[23758]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:09.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:09 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3501087460"}]': finished 2024-09-13T23:13:09.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:09 smithi022 ceph-mon[23758]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T23:13:09.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:09 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2403409397' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/282633179"}]: dispatch 2024-09-13T23:13:09.287 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:09 smithi076 podman[48294]: 2024-09-13 23:13:09.111858005 +0000 UTC m=+0.481853463 container cleanup 6ee1b8ce1803913235dfe6676635a2582db9fd91724d8033273437f7cba5b822 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T23:13:09.287 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:09 smithi076 bash[48294]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T23:13:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:09 smithi076 ceph-mon[24587]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T23:13:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:09 smithi076 ceph-mon[24587]: Deploying daemon node-exporter.b on smithi076 2024-09-13T23:13:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:09 smithi076 ceph-mon[24587]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:09 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3501087460"}]': finished 2024-09-13T23:13:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:09 smithi076 ceph-mon[24587]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T23:13:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:09 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2403409397' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/282633179"}]: dispatch 2024-09-13T23:13:09.768 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:09 smithi076 podman[48306]: 2024-09-13 23:13:09.388007259 +0000 UTC m=+0.643872281 container remove 6ee1b8ce1803913235dfe6676635a2582db9fd91724d8033273437f7cba5b822 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T23:13:09.769 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:09 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T23:13:09.769 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:09 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T23:13:09.769 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:09 smithi076 systemd[1]: Stopped Ceph node-exporter.b for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:13:09.769 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:09 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 4.279s CPU time. 2024-09-13T23:13:10.037 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:09 smithi076 systemd[1]: Starting Ceph node-exporter.b for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:13:10.037 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:09 smithi076 bash[48434]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-13T23:13:10.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:10 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2403409397' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/282633179"}]': finished 2024-09-13T23:13:10.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:10 smithi076 ceph-mon[24587]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T23:13:10.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:10 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/531123593' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1179215007"}]: dispatch 2024-09-13T23:13:10.363 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:10 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1179215007"}]: dispatch 2024-09-13T23:13:10.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:10 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2403409397' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/282633179"}]': finished 2024-09-13T23:13:10.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:10 smithi022 ceph-mon[28440]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T23:13:10.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:10 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/531123593' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1179215007"}]: dispatch 2024-09-13T23:13:10.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:10 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1179215007"}]: dispatch 2024-09-13T23:13:10.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:10 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2403409397' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/282633179"}]': finished 2024-09-13T23:13:10.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:10 smithi022 ceph-mon[23758]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T23:13:10.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:10 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/531123593' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1179215007"}]: dispatch 2024-09-13T23:13:10.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:10 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1179215007"}]: dispatch 2024-09-13T23:13:10.786 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:10 smithi076 bash[48434]: Getting image source signatures 2024-09-13T23:13:10.787 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:10 smithi076 bash[48434]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-13T23:13:10.787 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:10 smithi076 bash[48434]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-13T23:13:10.787 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:10 smithi076 bash[48434]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-13T23:13:11.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:11 smithi022 ceph-mon[28440]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:13:11.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:11 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1179215007"}]': finished 2024-09-13T23:13:11.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:11 smithi022 ceph-mon[28440]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T23:13:11.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:11 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/991244314' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3375084674"}]: dispatch 2024-09-13T23:13:11.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:11 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3375084674"}]: dispatch 2024-09-13T23:13:11.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:11 smithi022 ceph-mon[23758]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:13:11.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:11 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1179215007"}]': finished 2024-09-13T23:13:11.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:11 smithi022 ceph-mon[23758]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T23:13:11.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:11 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/991244314' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3375084674"}]: dispatch 2024-09-13T23:13:11.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:11 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3375084674"}]: dispatch 2024-09-13T23:13:11.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:11 smithi076 ceph-mon[24587]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:13:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:11 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1179215007"}]': finished 2024-09-13T23:13:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:11 smithi076 ceph-mon[24587]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T23:13:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:11 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/991244314' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3375084674"}]: dispatch 2024-09-13T23:13:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:11 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3375084674"}]: dispatch 2024-09-13T23:13:12.037 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:11 smithi076 bash[48434]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-13T23:13:12.037 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:11 smithi076 bash[48434]: Writing manifest to image destination 2024-09-13T23:13:12.378 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 podman[48434]: 2024-09-13 23:13:12.100664162 +0000 UTC m=+2.231689604 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T23:13:12.378 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 podman[48434]: 2024-09-13 23:13:12.249756936 +0000 UTC m=+2.380782365 container create 1141a03b8cd726af8bf72b48b61aea7383922413717234d32138fcfc3a58e1a5 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T23:13:12.379 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3375084674"}]': finished 2024-09-13T23:13:12.379 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-mon[24587]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T23:13:12.379 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/393358916' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3501087460"}]: dispatch 2024-09-13T23:13:12.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:12 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3375084674"}]': finished 2024-09-13T23:13:12.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:12 smithi022 ceph-mon[28440]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T23:13:12.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:12 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/393358916' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3501087460"}]: dispatch 2024-09-13T23:13:12.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:12 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3375084674"}]': finished 2024-09-13T23:13:12.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:12 smithi022 ceph-mon[23758]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T23:13:12.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:12 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/393358916' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3501087460"}]: dispatch 2024-09-13T23:13:13.037 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 podman[48434]: 2024-09-13 23:13:12.779292182 +0000 UTC m=+2.910317611 container init 1141a03b8cd726af8bf72b48b61aea7383922413717234d32138fcfc3a58e1a5 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T23:13:13.037 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 podman[48434]: 2024-09-13 23:13:12.782789242 +0000 UTC m=+2.913814680 container start 1141a03b8cd726af8bf72b48b61aea7383922413717234d32138fcfc3a58e1a5 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T23:13:13.037 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.787Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T23:13:13.037 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.787Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T23:13:13.037 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.787Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T23:13:13.038 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.787Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T23:13:13.038 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.788Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T23:13:13.038 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.788Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T23:13:13.038 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.788Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T23:13:13.038 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T23:13:13.038 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T23:13:13.038 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T23:13:13.038 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T23:13:13.038 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T23:13:13.038 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T23:13:13.038 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T23:13:13.039 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T23:13:13.039 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T23:13:13.039 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T23:13:13.039 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T23:13:13.039 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T23:13:13.039 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T23:13:13.039 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T23:13:13.039 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T23:13:13.040 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T23:13:13.040 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T23:13:13.040 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T23:13:13.040 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T23:13:13.040 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T23:13:13.040 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T23:13:13.040 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T23:13:13.040 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T23:13:13.040 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T23:13:13.040 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T23:13:13.040 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T23:13:13.041 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T23:13:13.042 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T23:13:13.042 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T23:13:13.042 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T23:13:13.042 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T23:13:13.042 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T23:13:13.042 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T23:13:13.042 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b[48520]: ts=2024-09-13T23:13:12.789Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T23:13:13.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:13 smithi022 ceph-mon[23758]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T23:13:13.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:13 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/393358916' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3501087460"}]': finished 2024-09-13T23:13:13.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:13 smithi022 ceph-mon[23758]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T23:13:13.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:13 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2614528645' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3357082900"}]: dispatch 2024-09-13T23:13:13.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:13 smithi022 ceph-mon[28440]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T23:13:13.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:13 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/393358916' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3501087460"}]': finished 2024-09-13T23:13:13.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:13 smithi022 ceph-mon[28440]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T23:13:13.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:13 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2614528645' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3357082900"}]: dispatch 2024-09-13T23:13:13.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:13 smithi076 ceph-mon[24587]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T23:13:13.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:13 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/393358916' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3501087460"}]': finished 2024-09-13T23:13:13.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:13 smithi076 ceph-mon[24587]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T23:13:13.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:13 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2614528645' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3357082900"}]: dispatch 2024-09-13T23:13:14.036 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:13 smithi076 bash[48434]: 1141a03b8cd726af8bf72b48b61aea7383922413717234d32138fcfc3a58e1a5 2024-09-13T23:13:14.037 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:13:13 smithi076 systemd[1]: Started Ceph node-exporter.b for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:13:14.407 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:14 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2614528645' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3357082900"}]': finished 2024-09-13T23:13:14.407 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:14 smithi076 ceph-mon[24587]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T23:13:14.408 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:14 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/109984306' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2824990478"}]: dispatch 2024-09-13T23:13:14.408 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:14 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:14.408 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:14 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:14.408 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:14 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:14.408 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:14 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:14.408 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:14 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T23:13:14.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2614528645' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3357082900"}]': finished 2024-09-13T23:13:14.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[23758]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T23:13:14.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/109984306' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2824990478"}]: dispatch 2024-09-13T23:13:14.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:14.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:14.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:14.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:14.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T23:13:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2614528645' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3357082900"}]': finished 2024-09-13T23:13:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[28440]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T23:13:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/109984306' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2824990478"}]: dispatch 2024-09-13T23:13:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:14 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T23:13:15.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[23758]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T23:13:15.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[23758]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T23:13:15.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T23:13:15.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[23758]: Reconfiguring daemon grafana.a on smithi076 2024-09-13T23:13:15.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[23758]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T23:13:15.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/109984306' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2824990478"}]': finished 2024-09-13T23:13:15.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[23758]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T23:13:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[28440]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T23:13:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[28440]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T23:13:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T23:13:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[28440]: Reconfiguring daemon grafana.a on smithi076 2024-09-13T23:13:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[28440]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T23:13:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/109984306' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2824990478"}]': finished 2024-09-13T23:13:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:15 smithi022 ceph-mon[28440]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T23:13:15.443 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:15 smithi076 ceph-mon[24587]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T23:13:15.443 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:15 smithi076 ceph-mon[24587]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T23:13:15.443 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:15 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T23:13:15.444 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:15 smithi076 ceph-mon[24587]: Reconfiguring daemon grafana.a on smithi076 2024-09-13T23:13:15.444 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:15 smithi076 ceph-mon[24587]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T23:13:15.444 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:15 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/109984306' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2824990478"}]': finished 2024-09-13T23:13:15.444 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:15 smithi076 ceph-mon[24587]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T23:13:16.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:16 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:13:16.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:16 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:13:16.443 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:16 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:13:17.037 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:16 smithi076 systemd[1]: Stopping Ceph grafana.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:13:17.037 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[40691]: t=2024-09-13T23:13:16+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-09-13T23:13:17.037 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:16 smithi076 podman[48696]: 2024-09-13 23:13:16.86024051 +0000 UTC m=+0.099740399 container died c818389ee4e7a2a5c692ed6bb3d721f1c5615dd5cec36a2613efe839a4eb5a03 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, name=ubi8, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.openshift.tags=base rhel8, vcs-type=git, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, release=236.1648460182, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, maintainer=Paul Cuzner , distribution-scope=public, version=8.5) 2024-09-13T23:13:17.392 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-mon[24587]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:13:17.393 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 podman[48696]: 2024-09-13 23:13:17.191386783 +0000 UTC m=+0.430886712 container cleanup c818389ee4e7a2a5c692ed6bb3d721f1c5615dd5cec36a2613efe839a4eb5a03 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.expose-services=, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, vcs-type=git, io.buildah.version=1.24.2) 2024-09-13T23:13:17.393 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 bash[48696]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a 2024-09-13T23:13:17.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:17 smithi022 ceph-mon[23758]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:13:17.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:17 smithi022 ceph-mon[28440]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:13:17.765 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 podman[48709]: 2024-09-13 23:13:17.392375802 +0000 UTC m=+0.530584835 container remove c818389ee4e7a2a5c692ed6bb3d721f1c5615dd5cec36a2613efe839a4eb5a03 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, name=ubi8, build-date=2022-03-28T10:36:18.413762, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, release=236.1648460182, io.buildah.version=1.24.2, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, description=Ceph Grafana Container, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, io.openshift.expose-services=, maintainer=Paul Cuzner , version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T23:13:17.765 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 bash[48723]: Error: no container with name or ID "ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana.a" found: no such container 2024-09-13T23:13:17.765 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-13T23:13:17.765 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 systemd[1]: Stopped Ceph grafana.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:13:17.765 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@grafana.a.service: Consumed 3.351s CPU time. 2024-09-13T23:13:17.765 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 systemd[1]: Starting Ceph grafana.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:13:17.766 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 podman[48783]: 2024-09-13 23:13:17.534732675 +0000 UTC m=+0.015623104 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-09-13T23:13:17.766 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 podman[48783]: 2024-09-13 23:13:17.634964855 +0000 UTC m=+0.115855320 container create 969941fc6e23fbe525c67332d529bb587176aeef7b9c656a6ad315cc61c2ea04 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, io.openshift.tags=base rhel8, architecture=x86_64, distribution-scope=public, com.redhat.component=ubi8-container, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, name=ubi8, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T23:13:18.036 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE Bus STOPPING 2024-09-13T23:13:18.037 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 podman[48783]: 2024-09-13 23:13:17.88742644 +0000 UTC m=+0.368316859 container init 969941fc6e23fbe525c67332d529bb587176aeef7b9c656a6ad315cc61c2ea04 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, build-date=2022-03-28T10:36:18.413762, architecture=x86_64, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, description=Ceph Grafana Container, name=ubi8, release=236.1648460182) 2024-09-13T23:13:18.037 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 podman[48783]: 2024-09-13 23:13:17.890395729 +0000 UTC m=+0.371286146 container start 969941fc6e23fbe525c67332d529bb587176aeef7b9c656a6ad315cc61c2ea04 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, io.buildah.version=1.24.2, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , release=236.1648460182, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, io.openshift.tags=base rhel8, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, version=8.5, vendor=Red Hat, Inc., name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container) 2024-09-13T23:13:18.037 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-09-13T23:13:18.037 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-09-13T23:13:18.037 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-09-13T23:13:18.037 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-09-13T23:13:18.037 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T23:13:18.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T23:13:18.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T23:13:18.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T23:13:18.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-09-13T23:13:18.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-09-13T23:13:18.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-09-13T23:13:18.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-09-13T23:13:18.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-09-13T23:13:18.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="App mode production" logger=settings 2024-09-13T23:13:18.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-09-13T23:13:18.038 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T23:13:18.039 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-09-13T23:13:18.039 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=539.481µs 2024-09-13T23:13:18.039 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-09-13T23:13:18.039 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-09-13T23:13:18.039 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-09-13T23:13:18.039 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 bash[48783]: 969941fc6e23fbe525c67332d529bb587176aeef7b9c656a6ad315cc61c2ea04 2024-09-13T23:13:18.039 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 systemd[1]: Started Ceph grafana.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:13:18.039 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-09-13T23:13:18.039 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-09-13T23:13:18.039 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:17+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-09-13T23:13:18.315 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T23:13:18.315 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE Bus STOPPED 2024-09-13T23:13:18.315 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE Bus STARTING 2024-09-13T23:13:18.315 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE Serving on http://:::9283 2024-09-13T23:13:18.316 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE Bus STARTED 2024-09-13T23:13:18.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-mon[24587]: from='client.25006 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:13:18.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T23:13:18.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T23:13:18.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:13:18.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi076.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T23:13:18.316 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.317 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:13:18.317 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi076.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T23:13:18.317 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.317 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:18+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-09-13T23:13:18.317 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:18+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T23:13:18.317 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:18+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-09-13T23:13:18.317 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:18+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-09-13T23:13:18.317 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:18+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T23:13:18.317 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:13:18+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-09-13T23:13:18.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[28440]: from='client.25006 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:13:18.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T23:13:18.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T23:13:18.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:13:18.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi076.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T23:13:18.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:13:18.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi076.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T23:13:18.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[23758]: from='client.25006 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:13:18.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T23:13:18.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T23:13:18.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:13:18.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi076.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T23:13:18.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:13:18.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi076.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T23:13:18.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:18 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:18.700 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE Bus STOPPING 2024-09-13T23:13:19.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T23:13:19.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE Bus STOPPED 2024-09-13T23:13:19.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE Bus STARTING 2024-09-13T23:13:19.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE Serving on http://:::9283 2024-09-13T23:13:19.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE Bus STARTED 2024-09-13T23:13:19.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:18] ENGINE Bus STOPPING 2024-09-13T23:13:19.376 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:19 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T23:13:19.377 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:19 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T23:13:19.377 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:19 smithi076 ceph-mon[24587]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:19.377 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:19 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:13:19.377 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:19 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi076.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T23:13:19.377 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:19 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:13:19.377 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:19 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi076.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T23:13:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T23:13:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T23:13:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[23758]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:13:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi076.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T23:13:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:13:19.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi076.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T23:13:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T23:13:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T23:13:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[28440]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:13:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi076.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T23:13:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:13:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:19 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi076.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T23:13:19.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:19 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T23:13:19.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:19 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:19] ENGINE Bus STOPPED 2024-09-13T23:13:19.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:19 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:19] ENGINE Bus STARTING 2024-09-13T23:13:19.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:19 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:19] ENGINE Serving on http://:::9283 2024-09-13T23:13:19.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:19 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:13:19] ENGINE Bus STARTED 2024-09-13T23:13:21.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:21 smithi022 ceph-mon[23758]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 646 B/s rd, 0 op/s 2024-09-13T23:13:21.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:21 smithi022 ceph-mon[28440]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 646 B/s rd, 0 op/s 2024-09-13T23:13:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:21 smithi076 ceph-mon[24587]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 646 B/s rd, 0 op/s 2024-09-13T23:13:22.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:21 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:13:21] "GET /metrics HTTP/1.1" 200 37879 "" "Prometheus/2.43.0" 2024-09-13T23:13:23.343 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T23:13:23.343 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.343 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.343 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.343 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.343 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:13:23.343 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:13:23.343 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.343 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:13:23.343 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:13:23.344 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:13:23.344 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:13:23.344 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[28440]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:13:23.344 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T23:13:23.344 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.344 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.344 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.344 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.344 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:13:23.344 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:13:23.344 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.345 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:13:23.345 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:13:23.345 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:13:23.345 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:13:23.345 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:23 smithi022 ceph-mon[23758]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:13:23.506 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T23:13:23.506 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.506 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.506 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.506 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.506 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:13:23.506 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:13:23.506 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:13:23.507 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:13:23.507 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:13:23.507 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:13:23.507 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:13:23.507 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-mon[24587]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:13:23.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:13:25.277 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:25 smithi022 ceph-mon[23758]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:13:25.277 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:25 smithi022 ceph-mon[28440]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:13:25.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-mon[24587]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:13:25.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:13:25.338+0000 7f403e8c2640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:13:25.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Creating ceph-iscsi config... 2024-09-13T23:13:25.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:13:25.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:13:25.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:13:25.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:25.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:25.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:13:25.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:13:25.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return _run_code(code, main_globals, None, 2024-09-13T23:13:25.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:13:25.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: exec(code, run_globals) 2024-09-13T23:13:25.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:13:25.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:13:25.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:13:25.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:13:25.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:13:25.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:13:25.539 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:13:25.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:25.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:25.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:13:25.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:13:25.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:13:25.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:13:25.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.wait_async( 2024-09-13T23:13:25.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:13:25.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:13:25.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:13:25.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return future.result(timeout) 2024-09-13T23:13:25.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:13:25.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.__get_result() 2024-09-13T23:13:25.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:13:25.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: raise self._exception 2024-09-13T23:13:25.541 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:13:25.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: out, err, code = await self._run_cephadm( 2024-09-13T23:13:25.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:13:25.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: raise OrchestratorError( 2024-09-13T23:13:25.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:13:25.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Creating ceph-iscsi config... 2024-09-13T23:13:25.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:13:25.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:13:25.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:13:25.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:25.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:25.542 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:13:25.543 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:13:25.543 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return _run_code(code, main_globals, None, 2024-09-13T23:13:25.543 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:13:25.543 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: exec(code, run_globals) 2024-09-13T23:13:25.543 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:13:25.543 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:13:25.544 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:13:25.544 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:13:25.544 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:13:25.544 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:13:25.544 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:13:25.544 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:25.544 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:25 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:26 smithi076 ceph-mon[24587]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:13:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: Creating ceph-iscsi config... 2024-09-13T23:13:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:13:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:13:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:13:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:13:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:13:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:13:26.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:13:26.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: exec(code, run_globals) 2024-09-13T23:13:26.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:13:26.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:13:26.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:13:26.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:13:26.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:13:26.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:13:26.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:13:26.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:26.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:26.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.wait_async( 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: return future.result(timeout) 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.__get_result() 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: raise self._exception 2024-09-13T23:13:26.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: out, err, code = await self._run_cephadm( 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: raise OrchestratorError( 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: Creating ceph-iscsi config... 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:13:26.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:13:26.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: exec(code, run_globals) 2024-09-13T23:13:26.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:13:26.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:13:26.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:13:26.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:13:26.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:13:26.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:13:26.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:13:26.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:26.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:26.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:26 smithi022 ceph-mon[23758]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:13:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.wait_async( 2024-09-13T23:13:26.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: return future.result(timeout) 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.__get_result() 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: raise self._exception 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: out, err, code = await self._run_cephadm( 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: raise OrchestratorError( 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:13:26.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:13:26.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:13:26.681 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:13:26.681 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:13:26.681 INFO:journalctl@ceph.mon.a.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:26.681 INFO:journalctl@ceph.mon.a.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:26.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:26 smithi022 ceph-mon[28440]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:13:26.681 INFO:journalctl@ceph.mon.c.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:13:26.681 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:13:26.681 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:13:26.681 INFO:journalctl@ceph.mon.c.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:13:26.681 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:26.681 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:13:26.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.wait_async( 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: return future.result(timeout) 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.__get_result() 2024-09-13T23:13:26.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: raise self._exception 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: out, err, code = await self._run_cephadm( 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: raise OrchestratorError( 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:13:26.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:13:26.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:13:26.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:13:26.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:13:26.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:13:26.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:13:26.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:13:26.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:13:26.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:13:26.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:13:26.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:13:26.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:13:26.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:13:27.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:27 smithi076 ceph-mon[24587]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-09-13T23:13:27.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:27 smithi022 ceph-mon[23758]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-09-13T23:13:27.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:27 smithi022 ceph-mon[28440]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-09-13T23:13:29.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:28 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:13:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:29 smithi076 ceph-mon[24587]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:13:29.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:29 smithi022 ceph-mon[23758]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:13:29.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:29 smithi022 ceph-mon[28440]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:13:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:31 smithi076 ceph-mon[24587]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:13:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:31 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:13:31.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:31 smithi022 ceph-mon[23758]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:13:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:31 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:13:31.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:31 smithi022 ceph-mon[28440]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:13:31.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:31 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:13:32.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:13:31] "GET /metrics HTTP/1.1" 200 37884 "" "Prometheus/2.43.0" 2024-09-13T23:13:33.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:32 smithi022 ceph-mon[28440]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:13:33.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:32 smithi022 ceph-mon[23758]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:13:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:32 smithi076 ceph-mon[24587]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:13:33.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:33.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:13:35.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:35 smithi022 ceph-mon[28440]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:35.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:35 smithi022 ceph-mon[23758]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:35.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:35 smithi076 ceph-mon[24587]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:37.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:37 smithi022 ceph-mon[23758]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:37.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:37 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2199160646' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:13:37.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:37 smithi022 ceph-mon[28440]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:37.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:37 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2199160646' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:13:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:37 smithi076 ceph-mon[24587]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:37 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2199160646' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:13:39.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:13:39.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:39 smithi022 ceph-mon[23758]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:39.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:39 smithi022 ceph-mon[28440]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:39.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:39 smithi076 ceph-mon[24587]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T23:13:41.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:41 smithi022 ceph-mon[23758]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:13:41.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:41 smithi022 ceph-mon[28440]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:13:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:41 smithi076 ceph-mon[24587]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:13:42.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:13:41] "GET /metrics HTTP/1.1" 200 37884 "" "Prometheus/2.43.0" 2024-09-13T23:13:43.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:42 smithi022 ceph-mon[23758]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:13:43.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:42 smithi022 ceph-mon[28440]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:13:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:42 smithi076 ceph-mon[24587]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:13:43.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:13:45.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:45 smithi022 ceph-mon[23758]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:13:45.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:45 smithi022 ceph-mon[28440]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:13:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:45 smithi076 ceph-mon[24587]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:13:46.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:46 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:13:46.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:46 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:13:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:46 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:13:47.349 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:47 smithi022 ceph-mon[23758]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:13:47.349 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:47 smithi022 ceph-mon[23758]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:13:47.349 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:47 smithi022 ceph-mon[28440]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:13:47.349 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:47 smithi022 ceph-mon[28440]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:13:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:47 smithi076 ceph-mon[24587]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:13:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:47 smithi076 ceph-mon[24587]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:13:49.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:13:49.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:49 smithi022 ceph-mon[23758]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:49.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:49 smithi022 ceph-mon[28440]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:49 smithi076 ceph-mon[24587]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:51.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:51 smithi022 ceph-mon[23758]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:51 smithi022 ceph-mon[28440]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:51 smithi076 ceph-mon[24587]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:52.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:13:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:13:51] "GET /metrics HTTP/1.1" 200 37883 "" "Prometheus/2.43.0" 2024-09-13T23:13:53.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:52 smithi022 ceph-mon[23758]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:13:53.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:52 smithi022 ceph-mon[28440]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:13:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:52 smithi076 ceph-mon[24587]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:13:53.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:53 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:13:55.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:55 smithi022 ceph-mon[23758]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:55.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:55 smithi022 ceph-mon[28440]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:55.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:55 smithi076 ceph-mon[24587]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:57 smithi022 ceph-mon[23758]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:57 smithi022 ceph-mon[23758]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:13:57.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:57 smithi022 ceph-mon[28440]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:57.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:57 smithi022 ceph-mon[28440]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:13:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:57 smithi076 ceph-mon[24587]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:13:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:57 smithi076 ceph-mon[24587]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:13:59.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:13:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:13:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:13:59.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:13:59 smithi022 ceph-mon[23758]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:13:59.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:13:59 smithi022 ceph-mon[28440]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:13:59.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:13:59 smithi076 ceph-mon[24587]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:01 smithi022 ceph-mon[23758]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:01 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:14:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:01 smithi022 ceph-mon[28440]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:01 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:14:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:01 smithi076 ceph-mon[24587]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:01 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:14:02.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:14:01] "GET /metrics HTTP/1.1" 200 37879 "" "Prometheus/2.43.0" 2024-09-13T23:14:03.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:02 smithi022 ceph-mon[28440]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:03.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:02 smithi022 ceph-mon[23758]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:02 smithi076 ceph-mon[24587]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:03.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:14:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:14:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:14:05.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:05 smithi022 ceph-mon[23758]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:05.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:05 smithi022 ceph-mon[28440]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:05 smithi076 ceph-mon[24587]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:07.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:07 smithi022 ceph-mon[23758]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:07.473 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:07 smithi022 ceph-mon[23758]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:14:07.474 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:07 smithi022 ceph-mon[28440]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:07.474 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:07 smithi022 ceph-mon[28440]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:14:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:07 smithi076 ceph-mon[24587]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:07 smithi076 ceph-mon[24587]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:14:09.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:14:08 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:14:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:14:09.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:09 smithi022 ceph-mon[23758]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:09.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:09 smithi022 ceph-mon[28440]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:09 smithi076 ceph-mon[24587]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:11.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:11 smithi022 ceph-mon[23758]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:11.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:11 smithi022 ceph-mon[28440]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:11 smithi076 ceph-mon[24587]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:12.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:14:11] "GET /metrics HTTP/1.1" 200 37879 "" "Prometheus/2.43.0" 2024-09-13T23:14:13.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:12 smithi022 ceph-mon[23758]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:13.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:12 smithi022 ceph-mon[28440]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:13.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:12 smithi076 ceph-mon[24587]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:13.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:14:13 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:14:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:14:15.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:15 smithi022 ceph-mon[23758]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:15 smithi022 ceph-mon[28440]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:15.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:15 smithi076 ceph-mon[24587]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:16.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:16 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:14:16.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:16 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:14:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:16 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:14:17.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:17 smithi022 ceph-mon[23758]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:17.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:17 smithi022 ceph-mon[23758]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:14:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:17 smithi022 ceph-mon[28440]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:17 smithi022 ceph-mon[28440]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:14:17.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:17 smithi076 ceph-mon[24587]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:17.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:17 smithi076 ceph-mon[24587]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:14:19.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:14:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:14:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:14:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:19 smithi022 ceph-mon[23758]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:19.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:19 smithi022 ceph-mon[28440]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:19 smithi076 ceph-mon[24587]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:21.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:21 smithi022 ceph-mon[23758]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:21.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:21 smithi022 ceph-mon[28440]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:21 smithi076 ceph-mon[24587]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:22.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:21 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:14:21] "GET /metrics HTTP/1.1" 200 37881 "" "Prometheus/2.43.0" 2024-09-13T23:14:23.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:22 smithi022 ceph-mon[23758]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:23.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:22 smithi022 ceph-mon[28440]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:23.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:22 smithi076 ceph-mon[24587]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:23.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:14:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:14:23.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:14:25.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:25 smithi022 ceph-mon[23758]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:25.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:25 smithi022 ceph-mon[28440]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:25 smithi076 ceph-mon[24587]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:27.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:27 smithi022 ceph-mon[23758]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:27.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:27 smithi022 ceph-mon[23758]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:14:27.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:27 smithi022 ceph-mon[28440]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:27.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:27 smithi022 ceph-mon[28440]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:14:27.437 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:27 smithi076 ceph-mon[24587]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:27.438 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:27 smithi076 ceph-mon[24587]: from='client.25069 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:14:28.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:14:28 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:14:28.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:14:29.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:29 smithi076 ceph-mon[24587]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:29.347 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:29 smithi022 ceph-mon[23758]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:29.348 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:29 smithi022 ceph-mon[28440]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:30.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:14:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:14:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:14:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:14:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:14:30.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:14:30.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:14:30.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:14:30.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:14:30.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:30 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:14:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:30 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:30 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:30 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:30 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:14:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:14:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:30 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:14:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:14:30.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:30 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:14:30.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:30 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:14:31.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:31 smithi022 ceph-mon[23758]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:31 smithi022 ceph-mon[23758]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:14:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:31 smithi022 ceph-mon[23758]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:14:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:31 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:14:31.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:31 smithi022 ceph-mon[28440]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:31.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:31 smithi022 ceph-mon[28440]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:14:31.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:31 smithi022 ceph-mon[28440]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:14:31.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:31 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:14:31.700 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:31 smithi076 ceph-mon[24587]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:31.700 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:31 smithi076 ceph-mon[24587]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:14:31.700 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:31 smithi076 ceph-mon[24587]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:14:31.700 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:31 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:14:32.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:14:31] "GET /metrics HTTP/1.1" 200 37881 "" "Prometheus/2.43.0" 2024-09-13T23:14:33.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:32 smithi022 ceph-mon[23758]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:33.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:32 smithi022 ceph-mon[28440]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:32 smithi076 ceph-mon[24587]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:14:33.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:14:33.084+0000 7f403e8c2640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:14:33.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Creating ceph-iscsi config... 2024-09-13T23:14:33.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:14:33.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:14:33.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:14:33.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:33.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:33.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:14:33.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:14:33.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return _run_code(code, main_globals, None, 2024-09-13T23:14:33.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:14:33.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: exec(code, run_globals) 2024-09-13T23:14:33.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:14:33.289 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:14:33.289 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:14:33.289 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:14:33.289 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:14:33.289 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:14:33.289 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:14:33.289 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:33.289 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:33.290 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:14:33.290 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:14:33.290 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:14:33.290 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:14:33.290 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.wait_async( 2024-09-13T23:14:33.290 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:14:33.290 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:14:33.290 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:14:33.290 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return future.result(timeout) 2024-09-13T23:14:33.290 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:14:33.290 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.__get_result() 2024-09-13T23:14:33.290 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: raise self._exception 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: out, err, code = await self._run_cephadm( 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: raise OrchestratorError( 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Creating ceph-iscsi config... 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:33.291 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return _run_code(code, main_globals, None, 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: exec(code, run_globals) 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:33.292 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:33.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:14:33.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:14:34.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:33 smithi022 ceph-mon[28440]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:14:34.176 INFO:journalctl@ceph.mon.c.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:14:34.176 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:14:34.176 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:14:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:14:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:14:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:14:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:14:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:14:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:14:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:14:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:14:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:14:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:14:34.178 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:14:34.178 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:14:34.178 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:14:34.178 INFO:journalctl@ceph.mon.c.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:34.178 INFO:journalctl@ceph.mon.c.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:34.178 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:14:34.178 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:14:34.178 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:14:34.178 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:14:34.178 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.wait_async( 2024-09-13T23:14:34.178 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:14:34.178 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: return future.result(timeout) 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.__get_result() 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: raise self._exception 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: out, err, code = await self._run_cephadm( 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: raise OrchestratorError( 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:14:34.179 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:14:34.180 INFO:journalctl@ceph.mon.c.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:14:34.180 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:34.180 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:34.180 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:14:34.180 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:14:34.180 INFO:journalctl@ceph.mon.c.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:14:34.180 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:14:34.180 INFO:journalctl@ceph.mon.c.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:14:34.180 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:14:34.180 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:14:34.180 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:14:34.180 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:14:34.181 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:14:34.181 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:14:34.181 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:14:34.181 INFO:journalctl@ceph.mon.c.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:34.181 INFO:journalctl@ceph.mon.c.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:34.181 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:33 smithi022 ceph-mon[23758]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:14:34.181 INFO:journalctl@ceph.mon.a.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:14:34.181 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:14:34.181 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:14:34.181 INFO:journalctl@ceph.mon.a.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:14:34.181 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:14:34.182 INFO:journalctl@ceph.mon.a.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.wait_async( 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: return future.result(timeout) 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.__get_result() 2024-09-13T23:14:34.183 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: raise self._exception 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: out, err, code = await self._run_cephadm( 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: raise OrchestratorError( 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:34.184 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:34.185 INFO:journalctl@ceph.mon.a.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:33 smithi076 ceph-mon[24587]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:14:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: Creating ceph-iscsi config... 2024-09-13T23:14:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:14:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:14:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:14:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:14:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:14:34.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:14:34.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:14:34.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: exec(code, run_globals) 2024-09-13T23:14:34.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:14:34.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:14:34.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:14:34.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:14:34.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:14:34.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:14:34.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:14:34.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:34.288 INFO:journalctl@ceph.mon.b.smithi076.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.wait_async( 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: return future.result(timeout) 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.__get_result() 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: raise self._exception 2024-09-13T23:14:34.289 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: out, err, code = await self._run_cephadm( 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: raise OrchestratorError( 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: Creating ceph-iscsi config... 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:14:34.290 INFO:journalctl@ceph.mon.b.smithi076.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:14:34.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:14:34.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: exec(code, run_globals) 2024-09-13T23:14:34.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:14:34.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:14:34.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:14:34.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:14:34.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:14:34.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:14:34.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:14:34.291 INFO:journalctl@ceph.mon.b.smithi076.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:14:34.292 INFO:journalctl@ceph.mon.b.smithi076.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:14:35.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:34 smithi022 ceph-mon[23758]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:35.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:34 smithi022 ceph-mon[28440]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:35.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:34 smithi076 ceph-mon[24587]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:37.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:37 smithi022 ceph-mon[23758]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-09-13T23:14:37.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:37 smithi022 ceph-mon[28440]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-09-13T23:14:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:37 smithi076 ceph-mon[24587]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-09-13T23:14:39.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:14:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:14:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:14:39.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:39 smithi022 ceph-mon[23758]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T23:14:39.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:39 smithi022 ceph-mon[28440]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T23:14:39.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:39 smithi076 ceph-mon[24587]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T23:14:41.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:41 smithi022 ceph-mon[23758]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T23:14:41.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:41 smithi022 ceph-mon[28440]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T23:14:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:41 smithi076 ceph-mon[24587]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T23:14:42.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:14:41] "GET /metrics HTTP/1.1" 200 37881 "" "Prometheus/2.43.0" 2024-09-13T23:14:43.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:42 smithi022 ceph-mon[23758]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T23:14:43.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:42 smithi022 ceph-mon[28440]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T23:14:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:42 smithi076 ceph-mon[24587]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T23:14:43.786 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:14:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:14:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:14:45.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:45 smithi022 ceph-mon[23758]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-13T23:14:45.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:45 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/604070872' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:14:45.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:45 smithi022 ceph-mon[28440]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-13T23:14:45.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:45 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/604070872' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:14:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:45 smithi076 ceph-mon[24587]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-13T23:14:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:45 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/604070872' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:14:46.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:46 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:14:46.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:46 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:14:46.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:46 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:14:47.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:47 smithi022 ceph-mon[23758]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T23:14:47.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:47 smithi022 ceph-mon[28440]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T23:14:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:47 smithi076 ceph-mon[24587]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T23:14:49.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:14:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:14:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:14:49.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:49 smithi022 ceph-mon[23758]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T23:14:49.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:49 smithi022 ceph-mon[28440]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T23:14:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:49 smithi076 ceph-mon[24587]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T23:14:51.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:51 smithi022 ceph-mon[23758]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T23:14:51.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:51 smithi022 ceph-mon[28440]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T23:14:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:51 smithi076 ceph-mon[24587]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T23:14:52.036 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:14:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:14:51] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-13T23:14:53.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:52 smithi022 ceph-mon[23758]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-09-13T23:14:53.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:52 smithi022 ceph-mon[28440]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-09-13T23:14:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:52 smithi076 ceph-mon[24587]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-09-13T23:14:53.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:14:53 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:14:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:14:55.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:55 smithi022 ceph-mon[23758]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-09-13T23:14:55.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:55 smithi022 ceph-mon[23758]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:14:55.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:55 smithi022 ceph-mon[28440]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-09-13T23:14:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:55 smithi022 ceph-mon[28440]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:14:55.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:55 smithi076 ceph-mon[24587]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-09-13T23:14:55.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:55 smithi076 ceph-mon[24587]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:14:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:57 smithi022 ceph-mon[23758]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T23:14:57.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:57 smithi022 ceph-mon[28440]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T23:14:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:57 smithi076 ceph-mon[24587]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T23:14:59.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:14:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:14:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:14:59.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:14:59 smithi022 ceph-mon[23758]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:59.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:14:59 smithi022 ceph-mon[28440]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:14:59.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:14:59 smithi076 ceph-mon[24587]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:01 smithi022 ceph-mon[23758]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:01 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:15:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:01 smithi022 ceph-mon[28440]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:01 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:15:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:01 smithi076 ceph-mon[24587]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:01.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:01 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:15:02.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:15:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:15:01] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-13T23:15:03.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:02 smithi022 ceph-mon[23758]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:03.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:02 smithi022 ceph-mon[28440]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:02 smithi076 ceph-mon[24587]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:03.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:15:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:15:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:15:05.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:05 smithi022 ceph-mon[23758]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:05.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:05 smithi022 ceph-mon[23758]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:05.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:05 smithi022 ceph-mon[28440]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:05.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:05 smithi022 ceph-mon[28440]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:05 smithi076 ceph-mon[24587]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:05 smithi076 ceph-mon[24587]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:07.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:07 smithi022 ceph-mon[23758]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:07 smithi022 ceph-mon[28440]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:07.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:07 smithi076 ceph-mon[24587]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:09.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:15:08 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:15:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:15:09.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:09 smithi022 ceph-mon[23758]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:09.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:09 smithi022 ceph-mon[28440]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:09 smithi076 ceph-mon[24587]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:11.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:11 smithi022 ceph-mon[23758]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:11.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:11 smithi022 ceph-mon[28440]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:11 smithi076 ceph-mon[24587]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:12.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:15:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:15:11] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-13T23:15:13.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:12 smithi022 ceph-mon[23758]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:13.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:12 smithi022 ceph-mon[28440]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:13.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:12 smithi076 ceph-mon[24587]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:13.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:15:13 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:15:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:15:15.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:15 smithi022 ceph-mon[23758]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:15.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:15 smithi022 ceph-mon[23758]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:15.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:15 smithi022 ceph-mon[28440]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:15 smithi022 ceph-mon[28440]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:15.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:15 smithi076 ceph-mon[24587]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:15.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:15 smithi076 ceph-mon[24587]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:16.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:16 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:15:16.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:16 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:15:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:16 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:15:17.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:17 smithi022 ceph-mon[23758]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:17.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:17 smithi022 ceph-mon[28440]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:17.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:17 smithi076 ceph-mon[24587]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:19.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:15:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:15:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:15:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:19 smithi022 ceph-mon[23758]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:19.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:19 smithi022 ceph-mon[28440]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:19 smithi076 ceph-mon[24587]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:21.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:21 smithi022 ceph-mon[23758]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:21.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:21 smithi022 ceph-mon[28440]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:21 smithi076 ceph-mon[24587]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:22.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:15:21 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:15:21] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.43.0" 2024-09-13T23:15:23.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:22 smithi022 ceph-mon[23758]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:23.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:22 smithi022 ceph-mon[28440]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:23.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:22 smithi076 ceph-mon[24587]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:23.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:15:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:15:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:15:25.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:25 smithi022 ceph-mon[23758]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:25.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:25 smithi022 ceph-mon[23758]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:25.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:25 smithi022 ceph-mon[28440]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:25 smithi022 ceph-mon[28440]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:25 smithi076 ceph-mon[24587]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:25 smithi076 ceph-mon[24587]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:27.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:27 smithi022 ceph-mon[23758]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:27.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:27 smithi022 ceph-mon[28440]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:27 smithi076 ceph-mon[24587]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:29.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:15:28 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:15:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:15:29.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:29 smithi022 ceph-mon[23758]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:29.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:29 smithi022 ceph-mon[28440]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:29 smithi076 ceph-mon[24587]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:31.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:31 smithi022 ceph-mon[23758]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:31.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:31 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:15:31.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:31 smithi022 ceph-mon[28440]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:31 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:15:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:31 smithi076 ceph-mon[24587]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:31 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:15:32.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:15:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:15:31] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-13T23:15:33.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:32 smithi022 ceph-mon[23758]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:33.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:32 smithi022 ceph-mon[28440]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:33.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:32 smithi076 ceph-mon[24587]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:33.291 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T23:15:33.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:15:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:15:33.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:15:35.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:35 smithi022 ceph-mon[28440]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:35.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:35 smithi022 ceph-mon[28440]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:35.318 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:35 smithi022 ceph-mon[23758]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:35.318 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:35 smithi022 ceph-mon[23758]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:35.489 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:35 smithi076 ceph-mon[24587]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:35.489 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:35 smithi076 ceph-mon[24587]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:35.885 INFO:teuthology.orchestra.run.smithi022.stdout:Scheduled to redeploy mgr.y on host 'smithi022' 2024-09-13T23:15:36.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:36 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2290082173' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T23:15:36.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:36 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:36.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:36 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:36.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:36 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:36.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:36 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2290082173' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T23:15:36.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:36 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:36.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:36 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:36.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:36 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:36 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2290082173' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T23:15:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:36 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:36 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:36 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:36.570 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps --refresh' 2024-09-13T23:15:37.175 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:37 smithi022 ceph-mon[28440]: from='client.15213 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:15:37.175 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:37 smithi022 ceph-mon[28440]: Schedule redeploy daemon mgr.y 2024-09-13T23:15:37.175 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:37 smithi022 ceph-mon[28440]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:37.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:37 smithi022 ceph-mon[23758]: from='client.15213 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:15:37.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:37 smithi022 ceph-mon[23758]: Schedule redeploy daemon mgr.y 2024-09-13T23:15:37.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:37 smithi022 ceph-mon[23758]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:37 smithi076 ceph-mon[24587]: from='client.15213 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:15:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:37 smithi076 ceph-mon[24587]: Schedule redeploy daemon mgr.y 2024-09-13T23:15:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:37 smithi076 ceph-mon[24587]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:38.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:15:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:15:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:15:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:15:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:15:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:15:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[23758]: Deploying daemon mgr.y on smithi022 2024-09-13T23:15:38.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:15:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:15:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:15:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:15:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:15:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:15:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:38 smithi022 ceph-mon[28440]: Deploying daemon mgr.y on smithi022 2024-09-13T23:15:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:38 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:15:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:38 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:15:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:15:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:38 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:15:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:38 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:15:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:38 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:15:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:38 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:15:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:38 smithi076 ceph-mon[24587]: Deploying daemon mgr.y on smithi022 2024-09-13T23:15:38.576 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:15:38.576 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (2m) 68s ago 10m 16.9M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:15:38.576 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (2m) 1s ago 10m 45.5M - 8.3.5 dad864ee21e9 969941fc6e23 2024-09-13T23:15:38.576 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (2m) 68s ago 9m 46.2M - 3.5 e1d6a67b021e 547e42ed0ec3 2024-09-13T23:15:38.576 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283 running (6m) 1s ago 15m 505M - 19.1.1-324-gecab7be5 0652ff2b5d78 194c1a9a8cc9 2024-09-13T23:15:38.576 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:9283 running (17m) 68s ago 17m 410M - 17.2.0 e1d6a67b021e 07f20d2b59b7 2024-09-13T23:15:38.576 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (17m) 68s ago 17m 56.3M 2048M 17.2.0 e1d6a67b021e 7d59bd9c71e8 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (15m) 1s ago 15m 49.5M 2048M 17.2.0 e1d6a67b021e a1194b4b1c43 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (15m) 68s ago 15m 43.2M 2048M 17.2.0 e1d6a67b021e 336a6f00a2a7 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (2m) 68s ago 11m 15.6M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (2m) 1s ago 11m 15.8M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (15m) 68s ago 15m 50.9M 2993M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (14m) 68s ago 14m 52.5M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (14m) 68s ago 14m 48.2M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (13m) 68s ago 13m 50.4M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (13m) 1s ago 13m 51.7M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (13m) 1s ago 13m 53.6M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (12m) 1s ago 12m 52.4M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (12m) 1s ago 12m 51.7M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:15:38.577 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (2m) 1s ago 11m 61.4M - 2.43.0 a07b618ecd1d 3a2530c356f9 2024-09-13T23:15:38.578 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (10m) 68s ago 10m 73.2M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:15:38.578 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (10m) 1s ago 10m 71.3M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:15:39.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:15:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:15:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:15:39.372 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 180' 2024-09-13T23:15:39.884 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:39 smithi022 ceph-mon[28440]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:39.884 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:39 smithi022 ceph-mon[28440]: from='client.25120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:15:39.885 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:39 smithi022 ceph-mon[23758]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:39.885 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:39 smithi022 ceph-mon[23758]: from='client.25120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:15:40.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:39 smithi076 ceph-mon[24587]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:40.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:39 smithi076 ceph-mon[24587]: from='client.25120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:15:41.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:41 smithi022 ceph-mon[23758]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:41.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:41 smithi022 ceph-mon[28440]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:42.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:41 smithi076 ceph-mon[24587]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:42.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:15:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:15:41] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-13T23:15:43.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:42 smithi022 ceph-mon[23758]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-13T23:15:43.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:42 smithi022 ceph-mon[28440]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-13T23:15:43.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:42 smithi076 ceph-mon[24587]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-13T23:15:43.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:15:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:15:43.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:15:45.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:45 smithi022 ceph-mon[23758]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-13T23:15:45.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:45 smithi022 ceph-mon[23758]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:45.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:45 smithi022 ceph-mon[28440]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-13T23:15:45.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:45 smithi022 ceph-mon[28440]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:45 smithi076 ceph-mon[24587]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-13T23:15:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:45 smithi076 ceph-mon[24587]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:46.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:46 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:15:46.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:46 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:15:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:46 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:15:47.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:47 smithi022 ceph-mon[23758]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-13T23:15:47.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:47 smithi022 ceph-mon[28440]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-13T23:15:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:47 smithi076 ceph-mon[24587]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-13T23:15:49.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:15:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:15:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:15:49.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:49 smithi022 ceph-mon[23758]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:49.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:49 smithi022 ceph-mon[28440]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:49 smithi076 ceph-mon[24587]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:51.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:51 smithi022 ceph-mon[23758]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:51.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:51 smithi022 ceph-mon[28440]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:51 smithi076 ceph-mon[24587]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:52.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:15:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:15:51] "GET /metrics HTTP/1.1" 200 37892 "" "Prometheus/2.43.0" 2024-09-13T23:15:53.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:52 smithi022 ceph-mon[23758]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:52 smithi022 ceph-mon[28440]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:53.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:52 smithi076 ceph-mon[24587]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T23:15:53.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:15:53 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:15:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:15:55.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:55 smithi022 ceph-mon[23758]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:55.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:55 smithi022 ceph-mon[23758]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:55.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:55 smithi022 ceph-mon[28440]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:55.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:55 smithi022 ceph-mon[28440]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:55.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:55 smithi076 ceph-mon[24587]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:55.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:55 smithi076 ceph-mon[24587]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:15:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:57 smithi022 ceph-mon[23758]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:57.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:57 smithi022 ceph-mon[28440]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:57 smithi076 ceph-mon[24587]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:15:59.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:15:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:15:58.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:15:59.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:15:59 smithi022 ceph-mon[23758]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:59.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:15:59 smithi022 ceph-mon[28440]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:15:59.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:15:59 smithi076 ceph-mon[24587]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:01 smithi022 ceph-mon[23758]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:01 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:16:01.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:01 smithi022 ceph-mon[28440]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:01 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:16:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:01 smithi076 ceph-mon[24587]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:01 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:16:02.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:16:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:16:01] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-13T23:16:02.923 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:02 smithi076 ceph-mon[24587]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:03.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:02 smithi022 ceph-mon[23758]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:03.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:02 smithi022 ceph-mon[28440]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:03.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:16:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:16:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:16:05.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:05 smithi022 ceph-mon[23758]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:05.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:05 smithi022 ceph-mon[23758]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:05.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:05 smithi022 ceph-mon[28440]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:05.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:05 smithi022 ceph-mon[28440]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:05 smithi076 ceph-mon[24587]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:05 smithi076 ceph-mon[24587]: from='client.15192 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:07.364 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:07 smithi022 ceph-mon[23758]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:07.364 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:07 smithi022 ceph-mon[28440]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:07 smithi076 ceph-mon[24587]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:08.827 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:08 smithi022 systemd[1]: Stopping Ceph mgr.y for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:16:09.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:16:08 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:16:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:16:09.112 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:08 smithi022 podman[61563]: 2024-09-13 23:16:08.826072299 +0000 UTC m=+0.136512378 container died 07f20d2b59b79619fc705742b216871bf40bbb8e763d6b1e4835da7f7ee5501f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, maintainer=Guillaume Abrioux , version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vendor=Red Hat, Inc., name=centos-stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-13T23:16:09.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:09 smithi022 ceph-mon[23758]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:09.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:09 smithi022 ceph-mon[28440]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:09.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:09 smithi022 podman[61563]: 2024-09-13 23:16:09.219834209 +0000 UTC m=+0.530274273 container cleanup 07f20d2b59b79619fc705742b216871bf40bbb8e763d6b1e4835da7f7ee5501f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y, architecture=x86_64, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-type=git, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, ceph=True, version=8, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.openshift.expose-services=, name=centos-stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T23:16:09.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:09 smithi022 bash[61563]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y 2024-09-13T23:16:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:09 smithi076 ceph-mon[24587]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:09.809 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:09 smithi022 podman[61575]: 2024-09-13 23:16:09.462960683 +0000 UTC m=+0.633893173 container remove 07f20d2b59b79619fc705742b216871bf40bbb8e763d6b1e4835da7f7ee5501f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, release=754, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vcs-type=git, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_CLEAN=True, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T23:16:09.810 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:09 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-13T23:16:09.810 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:09 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-13T23:16:09.810 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:09 smithi022 systemd[1]: Stopped Ceph mgr.y for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:16:09.810 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:09 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.y.service: Consumed 1min 4.959s CPU time. 2024-09-13T23:16:10.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:09 smithi022 systemd[1]: Starting Ceph mgr.y for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:16:10.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:10 smithi022 podman[61708]: 2024-09-13 23:16:09.923657443 +0000 UTC m=+0.016155484 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:16:10.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:10 smithi022 podman[61708]: 2024-09-13 23:16:10.039844892 +0000 UTC m=+0.132342974 container create 9d08241cbfbfde793dceef945accd6dc6f7611f88423a9670797591c9c70513f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-13T23:16:10.451 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:10 smithi022 podman[61708]: 2024-09-13 23:16:10.300653983 +0000 UTC m=+0.393152026 container init 9d08241cbfbfde793dceef945accd6dc6f7611f88423a9670797591c9c70513f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-13T23:16:10.451 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:10 smithi022 podman[61708]: 2024-09-13 23:16:10.303667672 +0000 UTC m=+0.396165712 container start 9d08241cbfbfde793dceef945accd6dc6f7611f88423a9670797591c9c70513f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-13T23:16:10.451 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:10 smithi022 ceph-mgr[61727]: -- 172.21.15.22:0/153176072 <== mon.1 v2:172.21.15.22:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b858f0d0e0 con 0x55b858eea800 2024-09-13T23:16:10.451 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:10 smithi022 bash[61708]: 9d08241cbfbfde793dceef945accd6dc6f7611f88423a9670797591c9c70513f 2024-09-13T23:16:10.451 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:10 smithi022 systemd[1]: Started Ceph mgr.y for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:16:10.750 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:10 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:10.449+0000 7fc9129cd040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T23:16:10.750 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:10 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:10.513+0000 7fc9129cd040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T23:16:10.750 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:10 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:10.674+0000 7fc9129cd040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T23:16:10.750 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:10 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:10.748+0000 7fc9129cd040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T23:16:11.071 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:10 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:10.915+0000 7fc9129cd040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:16:11.386 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:11.069+0000 7fc9129cd040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T23:16:11.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:11.416+0000 7fc9129cd040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T23:16:11.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:11.483+0000 7fc9129cd040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T23:16:11.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[23758]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[23758]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:16:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:16:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:16:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:16:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[23758]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:16:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[28440]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[28440]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:16:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:16:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:16:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:16:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-mon[28440]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:16:11.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:16:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:16:11] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-13T23:16:11.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:11 smithi076 ceph-mon[24587]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:11.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:11 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:11.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:11 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:11.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:11 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:11.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:11 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:11.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:11 smithi076 ceph-mon[24587]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:16:11.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:11 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:16:11.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:11 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:16:11.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:11 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:16:11.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:11 smithi076 ceph-mon[24587]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:16:12.070 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:11 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:11.689+0000 7fc9129cd040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T23:16:12.924 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:12.682+0000 7fc9129cd040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T23:16:12.924 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:12.754+0000 7fc9129cd040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T23:16:13.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:12 smithi022 ceph-mon[23758]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:13.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:12 smithi022 ceph-mon[28440]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:13.286 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:12 smithi076 ceph-mon[24587]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:13.726 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:13.554+0000 7fc9129cd040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T23:16:13.726 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:13.697+0000 7fc9129cd040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T23:16:13.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:16:13 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:16:13.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:16:13.980 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:13.852+0000 7fc9129cd040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:16:13.981 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:13.923+0000 7fc9129cd040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T23:16:14.351 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:14 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:14.089+0000 7fc9129cd040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T23:16:14.351 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:14 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:14.169+0000 7fc9129cd040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T23:16:14.351 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:14 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:14.235+0000 7fc9129cd040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T23:16:14.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:14 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:14.404+0000 7fc9129cd040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T23:16:14.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:14 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:14.475+0000 7fc9129cd040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T23:16:14.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:14 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:14.547+0000 7fc9129cd040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T23:16:14.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:14 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:14.651+0000 7fc9129cd040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T23:16:15.124 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:14 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:14.718+0000 7fc9129cd040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T23:16:15.125 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:14 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:14.790+0000 7fc9129cd040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T23:16:15.791 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[23758]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:15.792 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:15.792 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:15.792 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:16:15.792 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:16:15.792 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:16:15.792 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:16:15.792 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1580240511' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:16:15.792 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:16:15.793 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:15.682+0000 7fc9129cd040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T23:16:15.793 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:16:15] ENGINE Bus STARTING 2024-09-13T23:16:15.793 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: CherryPy Checker: 2024-09-13T23:16:15.793 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: The Application mounted at '' has an empty config. 2024-09-13T23:16:15.793 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:16:15.793 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[28440]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:15.793 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:15.793 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:15.793 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:16:15.793 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:16:15.794 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:16:15.794 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:16:15.794 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1580240511' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:16:15.794 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:16:15.942 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:15 smithi076 ceph-mon[24587]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:15.942 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:15 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:15.943 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:15 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:15.943 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:15 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:16:15.943 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:15 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:16:15.943 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:15 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:16:15.943 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:15 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:16:15.943 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:15 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1580240511' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:16:15.943 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:15 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:16:16.057 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:16:15] ENGINE Serving on http://:::9283 2024-09-13T23:16:16.057 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:16:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:16:15] ENGINE Bus STARTED 2024-09-13T23:16:16.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:16 smithi022 ceph-mon[23758]: Standby manager daemon y restarted 2024-09-13T23:16:16.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:16 smithi022 ceph-mon[23758]: Standby manager daemon y started 2024-09-13T23:16:16.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:16 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T23:16:16.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:16 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:16:16.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:16 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T23:16:16.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:16 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:16:16.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:16 smithi022 ceph-mon[28440]: Standby manager daemon y restarted 2024-09-13T23:16:16.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:16 smithi022 ceph-mon[28440]: Standby manager daemon y started 2024-09-13T23:16:16.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:16 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T23:16:16.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:16 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:16:16.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:16 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T23:16:16.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:16 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:16:16.995 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:16 smithi076 ceph-mon[24587]: Standby manager daemon y restarted 2024-09-13T23:16:16.995 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:16 smithi076 ceph-mon[24587]: Standby manager daemon y started 2024-09-13T23:16:16.995 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:16 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T23:16:16.995 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:16 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:16:16.995 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:16 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T23:16:16.996 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:16 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:16:17.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:17 smithi022 ceph-mon[23758]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:17.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:17 smithi022 ceph-mon[23758]: mgrmap e27: x(active, since 3m), standbys: y 2024-09-13T23:16:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:17 smithi022 ceph-mon[28440]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:17 smithi022 ceph-mon[28440]: mgrmap e27: x(active, since 3m), standbys: y 2024-09-13T23:16:18.036 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:17 smithi076 ceph-mon[24587]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:18.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:17 smithi076 ceph-mon[24587]: mgrmap e27: x(active, since 3m), standbys: y 2024-09-13T23:16:19.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:16:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:16:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:16:19.878 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:19 smithi022 ceph-mon[28440]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:16:19.878 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:19 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:19.878 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:19 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:19.878 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:19 smithi022 ceph-mon[23758]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:16:19.878 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:19 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:19.878 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:19 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:20.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:19 smithi076 ceph-mon[24587]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:16:20.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:19 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:20.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:19 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:21.868 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:21 smithi022 ceph-mon[23758]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:16:21.868 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:21 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:21.868 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:21 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:21.868 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:21 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:16:21.868 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:21 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:16:21.868 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:21 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:21.869 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:21 smithi022 ceph-mon[28440]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:16:21.869 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:21 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:21.869 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:21 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:21.869 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:21 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:16:21.869 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:21 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:16:21.869 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:21 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:22.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:16:21 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:16:21] "GET /metrics HTTP/1.1" 200 37892 "" "Prometheus/2.43.0" 2024-09-13T23:16:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:21 smithi076 ceph-mon[24587]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:16:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:21 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:21 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:21 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:16:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:21 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:16:22.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:21 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:16:23.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:22 smithi076 ceph-mon[24587]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:23.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:22 smithi022 ceph-mon[23758]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:23.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:22 smithi022 ceph-mon[28440]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:23.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:16:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:16:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:16:25.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:25 smithi022 ceph-mon[23758]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:16:25.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:25 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:25.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:25 smithi022 ceph-mon[28440]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:16:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:25 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:25 smithi076 ceph-mon[24587]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:16:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:25 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:27.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:27 smithi022 ceph-mon[23758]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:27.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:27 smithi022 ceph-mon[28440]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:27 smithi076 ceph-mon[24587]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:29.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:16:28 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:16:28.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:16:29.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:29 smithi022 ceph-mon[23758]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:29.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:29 smithi022 ceph-mon[28440]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:29 smithi076 ceph-mon[24587]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:31.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:31 smithi022 ceph-mon[23758]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:31.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:31 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:16:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:31 smithi022 ceph-mon[28440]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:31 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:16:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:31 smithi076 ceph-mon[24587]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:31 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:16:32.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:16:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:16:31] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.43.0" 2024-09-13T23:16:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:32 smithi076 ceph-mon[24587]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:33.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:32 smithi022 ceph-mon[28440]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:33.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:32 smithi022 ceph-mon[23758]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:33.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:16:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:16:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:16:35.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:35 smithi022 ceph-mon[23758]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:35.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:35 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:35.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:35 smithi022 ceph-mon[28440]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:35.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:35 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:35.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:35 smithi076 ceph-mon[24587]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:35.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:35 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:37.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:37 smithi022 ceph-mon[23758]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:37.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:37 smithi022 ceph-mon[28440]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:37.536 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:37 smithi076 ceph-mon[24587]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:39.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:16:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:16:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:16:39.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:39 smithi022 ceph-mon[23758]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:39.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:39 smithi022 ceph-mon[28440]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:39 smithi076 ceph-mon[24587]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:41.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:41 smithi022 ceph-mon[23758]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:41.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:41 smithi022 ceph-mon[28440]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:41 smithi076 ceph-mon[24587]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:42.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:16:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:16:41] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.43.0" 2024-09-13T23:16:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:42 smithi076 ceph-mon[24587]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:43.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:42 smithi022 ceph-mon[23758]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:43.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:42 smithi022 ceph-mon[28440]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:43.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:16:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:16:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:16:45.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:45 smithi022 ceph-mon[23758]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:45.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:45 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:45.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:45 smithi022 ceph-mon[28440]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:45.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:45 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:45 smithi076 ceph-mon[24587]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:45 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:46.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:46 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:16:46.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:46 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:16:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:46 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:16:47.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:47 smithi022 ceph-mon[23758]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:47.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:47 smithi022 ceph-mon[28440]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:47 smithi076 ceph-mon[24587]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:49.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:16:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:16:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:16:49.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:49 smithi022 ceph-mon[23758]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:49.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:49 smithi022 ceph-mon[28440]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:49 smithi076 ceph-mon[24587]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:51.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:51 smithi022 ceph-mon[23758]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:51.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:51 smithi022 ceph-mon[28440]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:51 smithi076 ceph-mon[24587]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:52.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:16:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:16:51] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-13T23:16:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:52 smithi076 ceph-mon[24587]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:53.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:52 smithi022 ceph-mon[23758]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:53.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:52 smithi022 ceph-mon[28440]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:53.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:16:53 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:16:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:16:55.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:55 smithi022 ceph-mon[23758]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:55.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:55 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:55 smithi022 ceph-mon[28440]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:55 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:55.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:55 smithi076 ceph-mon[24587]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:55.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:55 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:16:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:57 smithi022 ceph-mon[23758]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:57.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:57 smithi022 ceph-mon[28440]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:57 smithi076 ceph-mon[24587]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:16:59.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:16:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:16:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:16:59.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:16:59 smithi022 ceph-mon[23758]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:59.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:16:59 smithi022 ceph-mon[28440]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:16:59.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:16:59 smithi076 ceph-mon[24587]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:01 smithi022 ceph-mon[23758]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:01 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:17:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:01 smithi022 ceph-mon[28440]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:01 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:17:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:01 smithi076 ceph-mon[24587]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:01 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:17:02.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:17:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:17:01] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.43.0" 2024-09-13T23:17:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:02 smithi076 ceph-mon[24587]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:03.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:02 smithi022 ceph-mon[23758]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:03.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:02 smithi022 ceph-mon[28440]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:03.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:17:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:17:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:17:05.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:05 smithi022 ceph-mon[23758]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:05.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:05 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:05.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:05 smithi022 ceph-mon[28440]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:05.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:05 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:05 smithi076 ceph-mon[24587]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:05 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:07.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:07 smithi022 ceph-mon[23758]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:07.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:07 smithi022 ceph-mon[28440]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:07 smithi076 ceph-mon[24587]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:09.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:17:08 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:17:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:17:09.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:09 smithi022 ceph-mon[23758]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:09.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:09 smithi022 ceph-mon[28440]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:09 smithi076 ceph-mon[24587]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:11.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:11 smithi022 ceph-mon[23758]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:11.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:11 smithi022 ceph-mon[28440]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:11 smithi076 ceph-mon[24587]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:12.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:17:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:17:11] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.43.0" 2024-09-13T23:17:13.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:12 smithi076 ceph-mon[24587]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:13.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:12 smithi022 ceph-mon[23758]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:13.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:12 smithi022 ceph-mon[28440]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:13.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:17:13 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:17:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:17:15.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:15 smithi022 ceph-mon[23758]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:15.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:15 smithi022 ceph-mon[28440]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:15.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:15 smithi076 ceph-mon[24587]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:16.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:16 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:16.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:16 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:17:16.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:16 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:16.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:16 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:17:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:16 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:16 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:17:17.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:17 smithi022 ceph-mon[23758]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:17 smithi022 ceph-mon[28440]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:17.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:17 smithi076 ceph-mon[24587]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:19.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:17:18 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:17:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:17:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:19 smithi022 ceph-mon[23758]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:19.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:19 smithi022 ceph-mon[28440]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:19 smithi076 ceph-mon[24587]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:21.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:21 smithi022 ceph-mon[23758]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:21.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:21 smithi022 ceph-mon[28440]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:21 smithi076 ceph-mon[24587]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:22.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:17:21 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:17:21] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.43.0" 2024-09-13T23:17:23.270 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:22 smithi022 ceph-mon[28440]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:23.270 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:22 smithi022 ceph-mon[23758]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:23.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:22 smithi076 ceph-mon[24587]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:23.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:17:23 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:17:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:17:25.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:25 smithi022 ceph-mon[23758]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:25.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:25 smithi022 ceph-mon[28440]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:25 smithi076 ceph-mon[24587]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:26.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:26 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:26.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:26 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:17:26.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:26 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:17:26.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:26 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:26.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:26 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:17:26.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:26 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:17:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:26 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:26 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:17:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:26 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:17:27.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:27 smithi022 ceph-mon[23758]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:27.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:27 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:17:27.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:27 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:17:27.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:27 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:17:27.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:27 smithi022 ceph-mon[28440]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:27.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:27 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:17:27.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:27 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:17:27.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:27 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:17:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:27 smithi076 ceph-mon[24587]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:27 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:17:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:27 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:17:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:27 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:17:29.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:17:28 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:17:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:17:29.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:29 smithi022 ceph-mon[23758]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:29.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:29 smithi022 ceph-mon[28440]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:29 smithi076 ceph-mon[24587]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:31.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:31 smithi022 ceph-mon[23758]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:31.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:31 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:17:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:31 smithi022 ceph-mon[28440]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:31 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:17:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:31 smithi076 ceph-mon[24587]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:31 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:17:32.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:17:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:17:31] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.43.0" 2024-09-13T23:17:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:32 smithi076 ceph-mon[24587]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:33.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:32 smithi022 ceph-mon[23758]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:33.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:32 smithi022 ceph-mon[28440]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:33.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:17:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:17:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:17:35.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:35 smithi022 ceph-mon[23758]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:35.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:35 smithi022 ceph-mon[28440]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:35.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:35 smithi076 ceph-mon[24587]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:36.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:36 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:36.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:36 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:36 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:37.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:37 smithi022 ceph-mon[23758]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:37.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:37 smithi022 ceph-mon[28440]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:37 smithi076 ceph-mon[24587]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:39.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:17:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:17:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:17:39.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:39 smithi022 ceph-mon[23758]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:17:39.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:39 smithi022 ceph-mon[28440]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:17:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:39 smithi076 ceph-mon[24587]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:17:41.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:41 smithi022 ceph-mon[23758]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:41.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:41 smithi022 ceph-mon[28440]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:41 smithi076 ceph-mon[24587]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:42.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:17:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:17:41] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.43.0" 2024-09-13T23:17:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:42 smithi076 ceph-mon[24587]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:43.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:42 smithi022 ceph-mon[23758]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:43.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:42 smithi022 ceph-mon[28440]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:43.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:17:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:17:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:17:45.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:45 smithi022 ceph-mon[23758]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:17:45.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:45 smithi022 ceph-mon[28440]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:17:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:45 smithi076 ceph-mon[24587]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:17:46.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:46 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:46.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:46 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:17:46.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:46 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:46.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:46 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:17:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:46 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:46 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:17:47.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:47 smithi022 ceph-mon[23758]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:47.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:47 smithi022 ceph-mon[28440]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:47 smithi076 ceph-mon[24587]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:49.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:17:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:17:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"172.21.15.76:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:17:49.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:49 smithi022 ceph-mon[23758]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:49.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:49 smithi022 ceph-mon[28440]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:49 smithi076 ceph-mon[24587]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:51.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:51 smithi022 ceph-mon[23758]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:51 smithi022 ceph-mon[28440]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:51 smithi076 ceph-mon[24587]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:52.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:17:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:17:51] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.43.0" 2024-09-13T23:17:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:52 smithi076 ceph-mon[24587]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:53.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:52 smithi022 ceph-mon[23758]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:53.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:52 smithi022 ceph-mon[28440]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:53.787 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:17:53 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:17:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9adcac8c-7223-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T23:17:55.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:55 smithi022 ceph-mon[23758]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:55 smithi022 ceph-mon[28440]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:55.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:55 smithi076 ceph-mon[24587]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:56.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:56 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:56.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:56 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:56 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:17:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:57 smithi022 ceph-mon[23758]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:57.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:57 smithi022 ceph-mon[28440]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:57 smithi076 ceph-mon[24587]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:17:59.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:17:59 smithi022 ceph-mon[23758]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:59.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:17:59 smithi022 ceph-mon[28440]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:17:59.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:17:59 smithi076 ceph-mon[24587]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:01 smithi022 ceph-mon[23758]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:01 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:01 smithi022 ceph-mon[28440]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:01 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:01 smithi076 ceph-mon[24587]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:01 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:02.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:18:01] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.43.0" 2024-09-13T23:18:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:02 smithi076 ceph-mon[24587]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:03.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:02 smithi022 ceph-mon[23758]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:03.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:02 smithi022 ceph-mon[28440]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:05.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:05 smithi022 ceph-mon[28440]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:05.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:05 smithi022 ceph-mon[23758]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:05 smithi076 ceph-mon[24587]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:06.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:06 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:06.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:06 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:06.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:06 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:07.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:07 smithi022 ceph-mon[28440]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:07.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:07 smithi022 ceph-mon[23758]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:07 smithi076 ceph-mon[24587]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:09.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:09 smithi022 ceph-mon[28440]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:09.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:09 smithi022 ceph-mon[23758]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:09 smithi076 ceph-mon[24587]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:11.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:11 smithi022 ceph-mon[28440]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:11.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:11 smithi022 ceph-mon[23758]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:11 smithi076 ceph-mon[24587]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:12.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:18:11] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.43.0" 2024-09-13T23:18:13.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:12 smithi076 ceph-mon[24587]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:13.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:12 smithi022 ceph-mon[23758]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:13.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:12 smithi022 ceph-mon[28440]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:15.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:15 smithi022 ceph-mon[28440]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:15.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:15 smithi022 ceph-mon[23758]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:15.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:15 smithi076 ceph-mon[24587]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:16.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:16 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:16.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:16 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:16.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:16 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:16.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:16 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:16 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:16 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:17.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:17 smithi022 ceph-mon[23758]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:17 smithi022 ceph-mon[28440]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:17.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:17 smithi076 ceph-mon[24587]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:19 smithi022 ceph-mon[23758]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:19.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:19 smithi022 ceph-mon[28440]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:19 smithi076 ceph-mon[24587]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:21.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:21 smithi022 ceph-mon[23758]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:21.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:21 smithi022 ceph-mon[28440]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:21 smithi076 ceph-mon[24587]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:22.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:21 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:18:21] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.43.0" 2024-09-13T23:18:23.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:22 smithi076 ceph-mon[24587]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:23.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:22 smithi022 ceph-mon[23758]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:23.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:22 smithi022 ceph-mon[28440]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:25.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:25 smithi022 ceph-mon[23758]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:25 smithi022 ceph-mon[28440]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:25 smithi076 ceph-mon[24587]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:26.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:26 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:26.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:26 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:26 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:27.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:27 smithi022 ceph-mon[23758]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:27.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:27 smithi022 ceph-mon[28440]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:27 smithi076 ceph-mon[24587]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:29.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:29 smithi022 ceph-mon[23758]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:29.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:29 smithi022 ceph-mon[28440]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:29 smithi076 ceph-mon[24587]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:31.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:31 smithi022 ceph-mon[23758]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:31 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:31 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:18:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:31 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:18:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:31 smithi022 ceph-mon[28440]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:31 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:31 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:18:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:31 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:18:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:31 smithi076 ceph-mon[24587]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:31 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:31 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:18:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:31 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:18:32.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:18:31] "GET /metrics HTTP/1.1" 200 37882 "" "Prometheus/2.43.0" 2024-09-13T23:18:32.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:32 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:18:32.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:32 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:18:32.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:32 smithi022 ceph-mon[23758]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:18:32.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:32 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:18:32.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:32 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:18:32.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:32 smithi022 ceph-mon[28440]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:18:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:32 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:18:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:32 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:18:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:32 smithi076 ceph-mon[24587]: from='mgr.34808 ' entity='mgr.x' 2024-09-13T23:18:33.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:33 smithi022 ceph-mon[23758]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:33.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:33 smithi022 ceph-mon[28440]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:33.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:33 smithi076 ceph-mon[24587]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:35.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:35 smithi022 ceph-mon[23758]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:35.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:35 smithi022 ceph-mon[28440]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:35.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:35 smithi076 ceph-mon[24587]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:36 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:36.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:36 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:36.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:36 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:37 smithi076 ceph-mon[24587]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:37.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:37 smithi022 ceph-mon[23758]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:37 smithi022 ceph-mon[28440]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:39.478 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:39 smithi022 ceph-mon[23758]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:39.479 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:39 smithi022 ceph-mon[28440]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:39 smithi076 ceph-mon[24587]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:41.243 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:41 smithi022 ceph-mon[23758]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:41.243 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:41 smithi022 ceph-mon[28440]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:41 smithi076 ceph-mon[24587]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:41.998 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:18:42.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:18:41] "GET /metrics HTTP/1.1" 200 37882 "" "Prometheus/2.43.0" 2024-09-13T23:18:43.256 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:42 smithi022 ceph-mon[23758]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:43.256 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:42 smithi022 ceph-mon[28440]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:42 smithi076 ceph-mon[24587]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:43.979 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:18:43.980 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (5m) 13s ago 13m 24.7M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:18:43.980 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (5m) 2m ago 13m 46.4M - 8.3.5 dad864ee21e9 969941fc6e23 2024-09-13T23:18:43.980 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (2m) 13s ago 12m 50.9M - 3.5 e1d6a67b021e 796e5fbb8bb2 2024-09-13T23:18:43.980 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283 running (9m) 2m ago 18m 505M - 19.1.1-324-gecab7be5 0652ff2b5d78 194c1a9a8cc9 2024-09-13T23:18:43.980 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (2m) 13s ago 20m 438M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:18:43.980 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (20m) 13s ago 20m 59.9M 2048M 17.2.0 e1d6a67b021e 7d59bd9c71e8 2024-09-13T23:18:43.980 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (18m) 2m ago 18m 50.9M 2048M 17.2.0 e1d6a67b021e a1194b4b1c43 2024-09-13T23:18:43.980 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (18m) 13s ago 18m 41.4M 2048M 17.2.0 e1d6a67b021e 336a6f00a2a7 2024-09-13T23:18:43.980 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (6m) 13s ago 14m 16.2M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:18:43.980 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (5m) 2m ago 14m 15.9M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:18:43.980 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (18m) 13s ago 18m 52.1M 2993M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:18:43.981 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (17m) 13s ago 17m 53.7M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:18:43.981 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (17m) 13s ago 17m 49.9M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:18:43.981 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (16m) 13s ago 16m 51.1M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:18:43.981 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (16m) 2m ago 16m 52.2M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:18:43.981 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (16m) 2m ago 16m 54.5M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:18:43.981 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (15m) 2m ago 15m 53.3M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:18:43.981 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (15m) 2m ago 15m 52.6M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:18:43.981 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (5m) 2m ago 14m 61.5M - 2.43.0 a07b618ecd1d 3a2530c356f9 2024-09-13T23:18:43.981 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (13m) 13s ago 13m 74.3M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:18:43.981 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (13m) 2m ago 13m 71.5M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:18:44.748 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T23:18:45.341 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:45 smithi022 ceph-mon[23758]: from='client.15255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:18:45.341 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:45 smithi022 ceph-mon[23758]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:45.341 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:45 smithi022 ceph-mon[28440]: from='client.15255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:18:45.341 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:45 smithi022 ceph-mon[28440]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:45 smithi076 ceph-mon[24587]: from='client.15255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:18:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:45 smithi076 ceph-mon[24587]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:46.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:46 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:46.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:46 smithi022 ceph-mon[23758]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:46.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:46 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:46.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:46 smithi022 ceph-mon[28440]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:46 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:18:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:46 smithi076 ceph-mon[24587]: from='mgr.34808 172.21.15.76:0/2237030580' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:46.821 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:18:46.821 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:18:46.821 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T23:18:46.821 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:18:46.821 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:18:46.821 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:18:46.821 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:18:46.821 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:18:46.821 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T23:18:46.821 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:18:46.821 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": {}, 2024-09-13T23:18:46.822 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:18:46.822 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:18:46.822 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:18:46.822 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:18:46.822 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T23:18:46.822 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:18:46.822 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:18:46.822 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:18:47.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:47 smithi022 ceph-mon[28440]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:47.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:47 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/390338711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:18:47.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:47 smithi022 ceph-mon[23758]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:47.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:47 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/390338711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:18:47.468 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T23:18:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:47 smithi076 ceph-mon[24587]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:47 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/390338711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:18:49.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:49 smithi022 ceph-mon[23758]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:49.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:49 smithi022 ceph-mon[28440]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:49.486 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-09-13T23:18:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:49 smithi076 ceph-mon[24587]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:50.167 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph -s' 2024-09-13T23:18:50.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:50 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2200490838' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:18:50.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:50 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2200490838' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:18:50.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:50 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2200490838' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:18:51.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:51 smithi022 ceph-mon[23758]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:51.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:51 smithi022 ceph-mon[28440]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:51 smithi076 ceph-mon[24587]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:52.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:18:51] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.43.0" 2024-09-13T23:18:52.312 INFO:teuthology.orchestra.run.smithi022.stdout: cluster: 2024-09-13T23:18:52.312 INFO:teuthology.orchestra.run.smithi022.stdout: id: 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:18:52.312 INFO:teuthology.orchestra.run.smithi022.stdout: health: HEALTH_OK 2024-09-13T23:18:52.312 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:18:52.312 INFO:teuthology.orchestra.run.smithi022.stdout: services: 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: mgr: x(active, since 6m), standbys: y 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: data: 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: pools: 6 pools, 161 pgs 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: objects: 209 objects, 585 KiB 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: pgs: 161 active+clean 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: io: 2024-09-13T23:18:52.313 INFO:teuthology.orchestra.run.smithi022.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-13T23:18:52.314 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:18:52.877 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph mgr fail' 2024-09-13T23:18:53.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:52 smithi022 ceph-mon[23758]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:53.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:52 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2642398416' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T23:18:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:52 smithi022 ceph-mon[28440]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:52 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2642398416' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T23:18:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:52 smithi076 ceph-mon[24587]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:18:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:52 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2642398416' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T23:18:55.426 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:18:55] ENGINE Bus STOPPING 2024-09-13T23:18:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-mon[28440]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3966175886' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:18:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:18:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-mon[28440]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T23:18:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-mon[28440]: Standby manager daemon x started 2024-09-13T23:18:55.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-mon[23758]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:55.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3966175886' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:18:55.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:18:55.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-mon[23758]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T23:18:55.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-mon[23758]: Standby manager daemon x started 2024-09-13T23:18:55.529 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-mon[24587]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:18:55.529 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3966175886' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:18:55.529 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:18:55.530 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-mon[24587]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T23:18:55.530 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-mon[24587]: Standby manager daemon x started 2024-09-13T23:18:55.530 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:55.177+0000 7f40a95f5640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T23:18:55.530 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ignoring --setuser ceph since I am not root 2024-09-13T23:18:55.530 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ignoring --setgroup ceph since I am not root 2024-09-13T23:18:55.530 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-mgr[44261]: -- 172.21.15.76:0/1590403463 <== mon.2 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e638dc70e0 con 0x55e638da5400 2024-09-13T23:18:55.530 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:55.326+0000 7f50cd77a040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T23:18:55.530 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:55.382+0000 7f50cd77a040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T23:18:55.530 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:55.528+0000 7f50cd77a040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T23:18:55.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:55.595+0000 7f50cd77a040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T23:18:55.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:55.742+0000 7f50cd77a040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:18:55.809 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:18:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T23:18:55.809 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:18:55] ENGINE Bus STOPPED 2024-09-13T23:18:55.809 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:18:55] ENGINE Bus STARTING 2024-09-13T23:18:55.809 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:18:55] ENGINE Serving on http://:::9283 2024-09-13T23:18:55.809 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:18:55] ENGINE Bus STARTED 2024-09-13T23:18:55.842 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 180' 2024-09-13T23:18:56.171 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:55.880+0000 7f50cd77a040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T23:18:56.320 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T23:18:56.320 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: mgrmap e28: y(active, starting, since 0.290353s), standbys: x 2024-09-13T23:18:56.320 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:18:56.320 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:18:56.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:18:56.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:18:56.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: Manager daemon y is now available 2024-09-13T23:18:56.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:56.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:18:56.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:18:56.322 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T23:18:56.322 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: mgrmap e28: y(active, starting, since 0.290353s), standbys: x 2024-09-13T23:18:56.322 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:18:56.322 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:18:56.322 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:18:56.322 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:18:56.322 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: Manager daemon y is now available 2024-09-13T23:18:56.323 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:56.324 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:18:56.324 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:18:56.481 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:56.208+0000 7f50cd77a040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T23:18:56.481 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:56.270+0000 7f50cd77a040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T23:18:56.481 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:56.480+0000 7f50cd77a040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T23:18:56.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T23:18:56.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: mgrmap e28: y(active, starting, since 0.290353s), standbys: x 2024-09-13T23:18:56.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:18:56.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:18:56.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:18:56.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:18:56.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:18:56.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:18:56.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:18:56.482 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:18:56.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:18:56.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:18:56.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:18:56.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:18:56.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:18:56.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:18:56.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:18:56.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:18:56.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: Manager daemon y is now available 2024-09-13T23:18:56.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:18:56.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:18:56.483 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:18:57.455 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-mon[23758]: mgrmap e29: y(active, since 1.2995s), standbys: x 2024-09-13T23:18:57.455 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-mon[23758]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:18:57.455 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-mon[28440]: mgrmap e29: y(active, since 1.2995s), standbys: x 2024-09-13T23:18:57.455 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-mon[28440]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:18:57.456 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:18:57.342+0000 7fc86f1c7640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:23:18:57] ENGINE Error in HTTPServer.serve 2024-09-13T23:18:57.456 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: Traceback (most recent call last): 2024-09-13T23:18:57.456 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:18:57.456 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: self._connections.run(self.expiration_interval) 2024-09-13T23:18:57.456 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:18:57.456 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: self._run(expiration_interval) 2024-09-13T23:18:57.456 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:18:57.456 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:18:57.456 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:18:57.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:18:57.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:18:57.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: s = self.context.wrap_socket( 2024-09-13T23:18:57.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:18:57.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: return self.sslsocket_class._create( 2024-09-13T23:18:57.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:18:57.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: self.do_handshake() 2024-09-13T23:18:57.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:18:57.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: self._sslobj.do_handshake() 2024-09-13T23:18:57.458 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:18:57.458 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:18:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:18:57.469 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:57 smithi076 ceph-mon[24587]: mgrmap e29: y(active, since 1.2995s), standbys: x 2024-09-13T23:18:57.469 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:57 smithi076 ceph-mon[24587]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:18:57.469 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:57.397+0000 7f50cd77a040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T23:18:57.469 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:57.468+0000 7f50cd77a040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T23:18:58.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:58 smithi022 ceph-mon[23758]: [13/Sep/2024:23:18:57] ENGINE Bus STARTING 2024-09-13T23:18:58.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:58 smithi022 ceph-mon[23758]: [13/Sep/2024:23:18:57] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:18:58.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:58 smithi022 ceph-mon[23758]: [13/Sep/2024:23:18:57] ENGINE Error in HTTPServer.serve 2024-09-13T23:18:58.426 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:18:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:18:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:18:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:18:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._run(expiration_interval) 2024-09-13T23:18:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:18:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:18:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:18:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:18:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:18:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: s = self.context.wrap_socket( 2024-09-13T23:18:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:18:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.sslsocket_class._create( 2024-09-13T23:18:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:18:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.do_handshake() 2024-09-13T23:18:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:18:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._sslobj.do_handshake() 2024-09-13T23:18:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:18:58.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:58 smithi022 ceph-mon[28440]: [13/Sep/2024:23:18:57] ENGINE Bus STARTING 2024-09-13T23:18:58.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:58 smithi022 ceph-mon[28440]: [13/Sep/2024:23:18:57] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:18:58.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:58 smithi022 ceph-mon[28440]: [13/Sep/2024:23:18:57] ENGINE Error in HTTPServer.serve 2024-09-13T23:18:58.428 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:18:58.428 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:18:58.428 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._run(expiration_interval) 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: s = self.context.wrap_socket( 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.sslsocket_class._create( 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.do_handshake() 2024-09-13T23:18:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:18:58.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._sslobj.do_handshake() 2024-09-13T23:18:58.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:18:58.505 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:58.235+0000 7f50cd77a040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T23:18:58.505 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:58.359+0000 7f50cd77a040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T23:18:58.506 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:58 smithi076 ceph-mon[24587]: [13/Sep/2024:23:18:57] ENGINE Bus STARTING 2024-09-13T23:18:58.506 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:58 smithi076 ceph-mon[24587]: [13/Sep/2024:23:18:57] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:18:58.506 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:58 smithi076 ceph-mon[24587]: [13/Sep/2024:23:18:57] ENGINE Error in HTTPServer.serve 2024-09-13T23:18:58.506 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:18:58.506 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:18:58.506 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:18:58.506 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:18:58.507 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._run(expiration_interval) 2024-09-13T23:18:58.507 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:18:58.507 INFO:journalctl@ceph.mon.b.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:18:58.507 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:18:58.507 INFO:journalctl@ceph.mon.b.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:18:58.507 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:18:58.507 INFO:journalctl@ceph.mon.b.smithi076.stdout: s = self.context.wrap_socket( 2024-09-13T23:18:58.507 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:18:58.507 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.sslsocket_class._create( 2024-09-13T23:18:58.507 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:18:58.507 INFO:journalctl@ceph.mon.b.smithi076.stdout: self.do_handshake() 2024-09-13T23:18:58.507 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:18:58.508 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._sslobj.do_handshake() 2024-09-13T23:18:58.508 INFO:journalctl@ceph.mon.b.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:18:58.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:58.503+0000 7f50cd77a040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:18:58.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:58.564+0000 7f50cd77a040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T23:18:58.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:58.717+0000 7f50cd77a040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T23:18:59.055 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:58.786+0000 7f50cd77a040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T23:18:59.055 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:58.860+0000 7f50cd77a040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T23:18:59.055 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:59.021+0000 7f50cd77a040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T23:18:59.312 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:59 smithi076 ceph-mon[24587]: [13/Sep/2024:23:18:57] ENGINE Serving on http://172.21.15.22:8765 2024-09-13T23:18:59.312 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:59 smithi076 ceph-mon[24587]: [13/Sep/2024:23:18:57] ENGINE Bus STARTED 2024-09-13T23:18:59.312 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:18:59 smithi076 ceph-mon[24587]: mgrmap e30: y(active, since 3s), standbys: x 2024-09-13T23:18:59.312 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:59.097+0000 7f50cd77a040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T23:18:59.312 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:59.164+0000 7f50cd77a040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T23:18:59.313 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:59.250+0000 7f50cd77a040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T23:18:59.313 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:59.311+0000 7f50cd77a040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T23:18:59.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:59 smithi022 ceph-mon[23758]: [13/Sep/2024:23:18:57] ENGINE Serving on http://172.21.15.22:8765 2024-09-13T23:18:59.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:59 smithi022 ceph-mon[23758]: [13/Sep/2024:23:18:57] ENGINE Bus STARTED 2024-09-13T23:18:59.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:18:59 smithi022 ceph-mon[23758]: mgrmap e30: y(active, since 3s), standbys: x 2024-09-13T23:18:59.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:59 smithi022 ceph-mon[28440]: [13/Sep/2024:23:18:57] ENGINE Serving on http://172.21.15.22:8765 2024-09-13T23:18:59.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:59 smithi022 ceph-mon[28440]: [13/Sep/2024:23:18:57] ENGINE Bus STARTED 2024-09-13T23:18:59.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:18:59 smithi022 ceph-mon[28440]: mgrmap e30: y(active, since 3s), standbys: x 2024-09-13T23:18:59.750 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:18:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:18:59.371+0000 7f50cd77a040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T23:19:00.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[28440]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:19:00.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[28440]: Standby manager daemon x restarted 2024-09-13T23:19:00.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[28440]: Standby manager daemon x started 2024-09-13T23:19:00.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:19:00.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:19:00.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:19:00.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:19:00.533 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[23758]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:19:00.533 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[23758]: Standby manager daemon x restarted 2024-09-13T23:19:00.533 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[23758]: Standby manager daemon x started 2024-09-13T23:19:00.533 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:19:00.533 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:19:00.533 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:19:00.533 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:00 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:19:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-mon[24587]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:19:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-mon[24587]: Standby manager daemon x restarted 2024-09-13T23:19:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-mon[24587]: Standby manager daemon x started 2024-09-13T23:19:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:19:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:19:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:19:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:19:00.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:19:00.190+0000 7f50cd77a040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T23:19:00.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:19:00] ENGINE Bus STARTING 2024-09-13T23:19:00.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: CherryPy Checker: 2024-09-13T23:19:00.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: The Application mounted at '' has an empty config. 2024-09-13T23:19:00.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:19:00.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:19:00] ENGINE Serving on http://:::9283 2024-09-13T23:19:00.538 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:19:00 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:19:00] ENGINE Bus STARTED 2024-09-13T23:19:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:01 smithi022 ceph-mon[23758]: mgrmap e31: y(active, since 5s), standbys: x 2024-09-13T23:19:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:01 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:01 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:01.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:01 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:01.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:01 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:01 smithi022 ceph-mon[28440]: mgrmap e31: y(active, since 5s), standbys: x 2024-09-13T23:19:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:01 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:01 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:01 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:01 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:01 smithi076 ceph-mon[24587]: mgrmap e31: y(active, since 5s), standbys: x 2024-09-13T23:19:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:01 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:01 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:01 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:01 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:02.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:19:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:19:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T23:19:02.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:02 smithi076 ceph-mon[24587]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:19:02.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:02 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:02.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:02 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:02.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:02 smithi022 ceph-mon[23758]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:19:02.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:02 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:02.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:02 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:02.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:02 smithi022 ceph-mon[28440]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:19:02.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:02 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:02.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:02 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:19:04.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:19:04.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:19:04.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:19:04.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:19:04.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:19:04.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:19:04.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T23:19:04.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:19:04.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:19:04.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:03 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:19:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:19:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:19:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:19:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:19:04.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:19:04.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:19:04.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T23:19:04.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:19:04.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:19:04.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:04.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:03 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:19:05.107 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:04 smithi022 ceph-mon[23758]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:19:05.107 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:04 smithi022 ceph-mon[23758]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:19:05.107 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:04 smithi022 ceph-mon[23758]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:19:05.107 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:04 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:19:05.107 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:04 smithi022 ceph-mon[23758]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:19:05.107 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:04 smithi022 ceph-mon[28440]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:19:05.107 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:04 smithi022 ceph-mon[28440]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:19:05.107 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:04 smithi022 ceph-mon[28440]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:19:05.108 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:04 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:19:05.108 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:04 smithi022 ceph-mon[28440]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:19:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:04 smithi076 ceph-mon[24587]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:19:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:04 smithi076 ceph-mon[24587]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:19:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:04 smithi076 ceph-mon[24587]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:19:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:04 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:19:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:04 smithi076 ceph-mon[24587]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:19:05.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:05 smithi022 ceph-mon[23758]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:05.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:05 smithi022 ceph-mon[23758]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T23:19:05.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:05 smithi022 ceph-mon[28440]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:05.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:05 smithi022 ceph-mon[28440]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T23:19:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:05 smithi076 ceph-mon[24587]: from='client.15240 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:05 smithi076 ceph-mon[24587]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T23:19:08.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:07 smithi022 ceph-mon[23758]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T23:19:08.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:07 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:08.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:07 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:08.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:07 smithi022 ceph-mon[28440]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T23:19:08.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:07 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:08.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:07 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:07 smithi076 ceph-mon[24587]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T23:19:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:07 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:07 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:08.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:07.895Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-13T23:19:08.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:07.895Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-13T23:19:08.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:07.895Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-13T23:19:08.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:07.896Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-13T23:19:08.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:07 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:07.896Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-13T23:19:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:08 smithi076 ceph-mon[24587]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:19:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:08 smithi076 ceph-mon[24587]: Reconfiguring daemon prometheus.a on smithi076 2024-09-13T23:19:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:08 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1557328747' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:19:09.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:08 smithi022 ceph-mon[23758]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:19:09.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:08 smithi022 ceph-mon[23758]: Reconfiguring daemon prometheus.a on smithi076 2024-09-13T23:19:09.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:08 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1557328747' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:19:09.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:08 smithi022 ceph-mon[28440]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:19:09.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:08 smithi022 ceph-mon[28440]: Reconfiguring daemon prometheus.a on smithi076 2024-09-13T23:19:09.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:08 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1557328747' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:19:10.071 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:09 smithi076 ceph-mon[24587]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:19:10.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:09 smithi022 ceph-mon[23758]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:19:10.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:09 smithi022 ceph-mon[28440]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:19:11.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:10 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:19:11.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 systemd[1]: Stopping Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:19:11.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:10 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:19:11.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:10 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:19:11.591 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:11.307Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T23:19:11.591 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:11.307Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T23:19:11.592 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:11.307Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T23:19:11.592 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:11.307Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T23:19:11.592 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:11.307Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T23:19:11.592 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:11.308Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T23:19:11.592 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:11.308Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T23:19:11.592 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:11.308Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T23:19:11.592 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:11.308Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T23:19:11.592 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:11.308Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T23:19:11.592 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:11.308Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T23:19:11.592 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[48129]: ts=2024-09-13T23:19:11.308Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T23:19:11.592 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 podman[53476]: 2024-09-13 23:19:11.318085849 +0000 UTC m=+0.116949098 container died 3a2530c356f9b9d7ebfdc61cadd96c4bd028761c0e022e4a67df147319f9c233 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:19:11.941 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 podman[53476]: 2024-09-13 23:19:11.681878815 +0000 UTC m=+0.480742069 container cleanup 3a2530c356f9b9d7ebfdc61cadd96c4bd028761c0e022e4a67df147319f9c233 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:19:11.941 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 bash[53476]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T23:19:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:11 smithi076 ceph-mon[24587]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:19:12.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:11 smithi076 podman[53490]: 2024-09-13 23:19:11.941354357 +0000 UTC m=+0.625248018 container remove 3a2530c356f9b9d7ebfdc61cadd96c4bd028761c0e022e4a67df147319f9c233 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:19:12.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T23:19:12.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 systemd[1]: Stopped Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:19:12.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 2.420s CPU time. 2024-09-13T23:19:12.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 systemd[1]: Starting Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:19:12.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:11 smithi022 ceph-mon[23758]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:19:12.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:11 smithi022 ceph-mon[28440]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:19:12.603 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 podman[53580]: 2024-09-13 23:19:12.308870342 +0000 UTC m=+0.101350858 container create f26b8c02ef4e78ffd7223403d746a6fa4e46add2db60e56e25e953fba5240036 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:19:12.603 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 podman[53580]: 2024-09-13 23:19:12.224239518 +0000 UTC m=+0.016720044 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T23:19:13.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 podman[53580]: 2024-09-13 23:19:12.603039459 +0000 UTC m=+0.395519975 container init f26b8c02ef4e78ffd7223403d746a6fa4e46add2db60e56e25e953fba5240036 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:19:13.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 podman[53580]: 2024-09-13 23:19:12.605968562 +0000 UTC m=+0.398449087 container start f26b8c02ef4e78ffd7223403d746a6fa4e46add2db60e56e25e953fba5240036 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.629Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.629Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.629Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi076 (none))" 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.629Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.629Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.630Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.630Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.631Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.631Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.634Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.634Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.342µs 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.634Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T23:19:13.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.640Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-13T23:19:13.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.653Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-13T23:19:13.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.662Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-13T23:19:13.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.663Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-13T23:19:13.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.663Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.62µs wal_replay_duration=28.575293ms wbl_replay_duration=100ns total_replay_duration=28.634449ms 2024-09-13T23:19:13.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.666Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T23:19:13.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.666Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T23:19:13.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.666Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T23:19:13.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.685Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.008332ms db_storage=814ns remote_storage=1.551µs web_handler=417ns query_engine=947ns scrape=405.277µs scrape_sd=101.466µs notify=16.423µs notify_sd=16.352µs rules=18.147839ms tracing=8.541µs 2024-09-13T23:19:13.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.685Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T23:19:13.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:19:12.685Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T23:19:13.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 bash[53580]: f26b8c02ef4e78ffd7223403d746a6fa4e46add2db60e56e25e953fba5240036 2024-09-13T23:19:13.039 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:19:12 smithi076 systemd[1]: Started Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:19:14.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:19:14.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:19:14.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: Adding iSCSI gateway http://:@172.21.15.22:5000 to Dashboard 2024-09-13T23:19:14.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:19:14.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:19:14.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:19:14.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:19:14.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:19:14.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:19:14.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:13 smithi076 ceph-mon[24587]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:19:14.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:19:14.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:19:14.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: Adding iSCSI gateway http://:@172.21.15.22:5000 to Dashboard 2024-09-13T23:19:14.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:19:14.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:19:14.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:19:14.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:19:14.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:19:14.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:19:14.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[23758]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:19:14.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:19:14.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:19:14.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: Adding iSCSI gateway http://:@172.21.15.22:5000 to Dashboard 2024-09-13T23:19:14.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:19:14.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:19:14.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:19:14.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:19:14.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:19:14.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:19:14.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:14.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:13 smithi022 ceph-mon[28440]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:19:16.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:16 smithi022 ceph-mon[28440]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:19:16.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:16 smithi022 ceph-mon[23758]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:19:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:16 smithi076 ceph-mon[24587]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:19:18.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:18 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:18.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:18 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:18.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:18 smithi022 ceph-mon[23758]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:18.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:18 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:18.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:18 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:18.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:18 smithi022 ceph-mon[28440]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:18 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:18 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:18 smithi076 ceph-mon[24587]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:19 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:19 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:19 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:19 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:19:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:19 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:19:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:19 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:19.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:19 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:19.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:19 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:19.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:19 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:19.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:19 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:19:19.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:19 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:19:19.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:19 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:19.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:19 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:19.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:19 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:19.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:19 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:19.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:19 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:19:19.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:19 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:19:19.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:19 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:19:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:20 smithi076 ceph-mon[24587]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:19:20.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:20 smithi022 ceph-mon[28440]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:19:20.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:20 smithi022 ceph-mon[23758]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:19:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:22 smithi076 ceph-mon[24587]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:19:22.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:22 smithi022 ceph-mon[23758]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:19:22.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:22 smithi022 ceph-mon[28440]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:19:22.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:19:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:19:22] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.43.0" 2024-09-13T23:19:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:24 smithi076 ceph-mon[24587]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:24.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:24 smithi022 ceph-mon[23758]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:24.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:24 smithi022 ceph-mon[28440]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:26 smithi076 ceph-mon[24587]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:26 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:19:26.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:26 smithi022 ceph-mon[23758]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:26 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:19:26.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:26 smithi022 ceph-mon[28440]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:26.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:26 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:19:28.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:27 smithi022 ceph-mon[28440]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:28.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:27 smithi022 ceph-mon[23758]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:27 smithi076 ceph-mon[24587]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:29.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:28 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:29.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:28 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:29.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:28 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:29 smithi076 ceph-mon[24587]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:30.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:29 smithi022 ceph-mon[28440]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:30.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:29 smithi022 ceph-mon[23758]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:32 smithi076 ceph-mon[24587]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:32.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:32 smithi022 ceph-mon[23758]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:32.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:32 smithi022 ceph-mon[28440]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:32.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:19:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:19:32] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.43.0" 2024-09-13T23:19:34.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:34 smithi076 ceph-mon[24587]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:34.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:34 smithi022 ceph-mon[23758]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:34.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:34 smithi022 ceph-mon[28440]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:36 smithi076 ceph-mon[24587]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:36.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:36 smithi022 ceph-mon[23758]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:36.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:36 smithi022 ceph-mon[28440]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:37 smithi022 ceph-mon[23758]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:38.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:37 smithi022 ceph-mon[28440]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:37 smithi076 ceph-mon[24587]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:39.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:38 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:39.338 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:38 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:39.339 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:38 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:39 smithi076 ceph-mon[24587]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:40.327 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:39 smithi022 ceph-mon[23758]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:40.328 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:39 smithi022 ceph-mon[28440]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:41.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:40 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:19:41.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:40 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:19:41.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:40 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:19:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:41 smithi076 ceph-mon[24587]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:42.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:41 smithi022 ceph-mon[23758]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:42.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:41 smithi022 ceph-mon[28440]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:42.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:19:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:19:42] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-13T23:19:44.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:44 smithi076 ceph-mon[24587]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:44.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:44 smithi022 ceph-mon[23758]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:44.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:44 smithi022 ceph-mon[28440]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:46 smithi076 ceph-mon[24587]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:46.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:46 smithi022 ceph-mon[23758]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:46.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:46 smithi022 ceph-mon[28440]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:48.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:47 smithi022 ceph-mon[23758]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:48.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:47 smithi022 ceph-mon[28440]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:48.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:47 smithi076 ceph-mon[24587]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:49.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:48 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:49.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:48 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:49.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:48 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:50.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:49 smithi022 ceph-mon[23758]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:50.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:49 smithi022 ceph-mon[28440]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:49 smithi076 ceph-mon[24587]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:52 smithi076 ceph-mon[24587]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:52.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:52 smithi022 ceph-mon[23758]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:52.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:19:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:19:52] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-13T23:19:52.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:52 smithi022 ceph-mon[28440]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:54 smithi076 ceph-mon[24587]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:54.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:54 smithi022 ceph-mon[23758]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:54.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:54 smithi022 ceph-mon[28440]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:56 smithi076 ceph-mon[24587]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:19:56.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:56 smithi022 ceph-mon[23758]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:56.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:19:56.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:56 smithi022 ceph-mon[28440]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:19:56.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:19:58.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:57 smithi022 ceph-mon[23758]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:58.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:57 smithi022 ceph-mon[28440]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:57 smithi076 ceph-mon[24587]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:19:59.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:58 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:59.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:58 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:19:59.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:58 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:19:59 smithi076 ceph-mon[24587]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:00.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:19:59 smithi022 ceph-mon[23758]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:00.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:19:59 smithi022 ceph-mon[28440]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:01.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:00 smithi076 ceph-mon[24587]: overall HEALTH_OK 2024-09-13T23:20:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:00 smithi022 ceph-mon[23758]: overall HEALTH_OK 2024-09-13T23:20:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:00 smithi022 ceph-mon[28440]: overall HEALTH_OK 2024-09-13T23:20:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:01 smithi076 ceph-mon[24587]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:02.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:01 smithi022 ceph-mon[23758]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:02.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:01 smithi022 ceph-mon[28440]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:02.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:20:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:20:02] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-13T23:20:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:04 smithi076 ceph-mon[24587]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:04.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:04 smithi022 ceph-mon[23758]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:04 smithi022 ceph-mon[28440]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:06.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:06 smithi076 ceph-mon[24587]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:06.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:06 smithi022 ceph-mon[23758]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:06.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:06 smithi022 ceph-mon[28440]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:07 smithi076 ceph-mon[24587]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:08.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:07 smithi022 ceph-mon[23758]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:08.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:07 smithi022 ceph-mon[28440]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:08 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:09.328 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:08 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:09.328 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:08 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:09 smithi076 ceph-mon[24587]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:10.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:09 smithi022 ceph-mon[23758]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:10.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:09 smithi022 ceph-mon[28440]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:11.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:10 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:20:11.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:10 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:20:11.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:10 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:20:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:11 smithi076 ceph-mon[24587]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:12.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:11 smithi022 ceph-mon[23758]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:12.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:11 smithi022 ceph-mon[28440]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:12.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:20:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:20:12] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.43.0" 2024-09-13T23:20:14.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:14 smithi076 ceph-mon[24587]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:14.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:14 smithi022 ceph-mon[23758]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:14.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:14 smithi022 ceph-mon[28440]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:16 smithi076 ceph-mon[24587]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:16.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:16 smithi022 ceph-mon[23758]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:16.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:16 smithi022 ceph-mon[28440]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:17 smithi076 ceph-mon[24587]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:18.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:17 smithi022 ceph-mon[23758]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:18.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:17 smithi022 ceph-mon[28440]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:18 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:18 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:20:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:18 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:20:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:18 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:20:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:18 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:20:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:18 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:20:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:18 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:19.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:18 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:20:19.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:18 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:20:19.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:18 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:20:19.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:18 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:20:19.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:18 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:20:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:18 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:18 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:20:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:18 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:20:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:18 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:20:19.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:18 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:20:19.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:18 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:20:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:19 smithi076 ceph-mon[24587]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:20.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:19 smithi022 ceph-mon[23758]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:20.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:19 smithi022 ceph-mon[28440]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:22 smithi076 ceph-mon[24587]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:22.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:22 smithi022 ceph-mon[23758]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:22.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:20:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:20:22] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.43.0" 2024-09-13T23:20:22.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:22 smithi022 ceph-mon[28440]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:24 smithi076 ceph-mon[24587]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:24.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:24 smithi022 ceph-mon[23758]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:24.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:24 smithi022 ceph-mon[28440]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:26 smithi076 ceph-mon[24587]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:26 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:20:26.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:26 smithi022 ceph-mon[23758]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:26.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:26 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:20:26.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:26 smithi022 ceph-mon[28440]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:26.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:26 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:20:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:27 smithi076 ceph-mon[24587]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:28.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:27 smithi022 ceph-mon[23758]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:28.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:27 smithi022 ceph-mon[28440]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:29.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:28 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:29.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:28 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:29.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:28 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:29 smithi076 ceph-mon[24587]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:30.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:29 smithi022 ceph-mon[23758]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:30.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:29 smithi022 ceph-mon[28440]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:32 smithi076 ceph-mon[24587]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:32.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:32 smithi022 ceph-mon[23758]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:32.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:32 smithi022 ceph-mon[28440]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:32.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:20:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:20:32] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-13T23:20:34.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:34 smithi076 ceph-mon[24587]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:34.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:34 smithi022 ceph-mon[23758]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:34.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:34 smithi022 ceph-mon[28440]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:36 smithi076 ceph-mon[24587]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:36.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:36 smithi022 ceph-mon[23758]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:36.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:36 smithi022 ceph-mon[28440]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:37 smithi076 ceph-mon[24587]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:38.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:37 smithi022 ceph-mon[23758]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:38.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:37 smithi022 ceph-mon[28440]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:39.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:38 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:39.327 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:38 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:39.328 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:38 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:40 smithi076 ceph-mon[24587]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:40.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:39 smithi022 ceph-mon[23758]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:40.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:40 smithi022 ceph-mon[28440]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:41.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:41 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:20:41.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:41 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:20:41.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:41 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:20:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:42 smithi076 ceph-mon[24587]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:42.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:42 smithi022 ceph-mon[23758]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:42.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:42 smithi022 ceph-mon[28440]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:42.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:20:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:20:42] "GET /metrics HTTP/1.1" 200 37892 "" "Prometheus/2.43.0" 2024-09-13T23:20:44.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:44 smithi076 ceph-mon[24587]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:44.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:44 smithi022 ceph-mon[23758]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:44.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:44 smithi022 ceph-mon[28440]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:46 smithi076 ceph-mon[24587]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:46.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:46 smithi022 ceph-mon[23758]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:46.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:46 smithi022 ceph-mon[28440]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:48.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:47 smithi076 ceph-mon[24587]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:47 smithi022 ceph-mon[23758]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:47 smithi022 ceph-mon[28440]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:49.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:48 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:49.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:48 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:49.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:48 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:50 smithi076 ceph-mon[24587]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:50.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:50 smithi022 ceph-mon[23758]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:50.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:50 smithi022 ceph-mon[28440]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:52 smithi076 ceph-mon[24587]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:52.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:52 smithi022 ceph-mon[28440]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:52.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:52 smithi022 ceph-mon[23758]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:52.680 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:20:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:20:52] "GET /metrics HTTP/1.1" 200 37892 "" "Prometheus/2.43.0" 2024-09-13T23:20:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:54 smithi076 ceph-mon[24587]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:54.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:54 smithi022 ceph-mon[28440]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:54.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:54 smithi022 ceph-mon[23758]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:56 smithi076 ceph-mon[24587]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:20:56.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:56 smithi022 ceph-mon[28440]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:56.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:20:56.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:56 smithi022 ceph-mon[23758]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:20:56.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:20:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:57 smithi076 ceph-mon[24587]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:58.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:57 smithi022 ceph-mon[28440]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:57 smithi022 ceph-mon[23758]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:20:59.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:20:58 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:59.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:20:58 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:20:59.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:20:58 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:00 smithi076 ceph-mon[24587]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:00.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:00 smithi022 ceph-mon[28440]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:00.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:00 smithi022 ceph-mon[23758]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:02.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:02 smithi076 ceph-mon[24587]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:02.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:21:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:21:02] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-13T23:21:02.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:02 smithi022 ceph-mon[23758]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:02.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:02 smithi022 ceph-mon[28440]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:04 smithi076 ceph-mon[24587]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:04.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:04 smithi022 ceph-mon[23758]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:04 smithi022 ceph-mon[28440]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:06.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:06 smithi076 ceph-mon[24587]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:06.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:06 smithi022 ceph-mon[23758]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:06.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:06 smithi022 ceph-mon[28440]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:08 smithi076 ceph-mon[24587]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:08.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:07 smithi022 ceph-mon[23758]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:08.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:07 smithi022 ceph-mon[28440]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:09.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:09 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:09.328 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:08 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:09.328 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:08 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:10 smithi076 ceph-mon[24587]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:10.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:10 smithi022 ceph-mon[23758]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:10.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:10 smithi022 ceph-mon[28440]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:11.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:11 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:21:11.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:11 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:21:11.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:11 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:21:12.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:12 smithi076 ceph-mon[24587]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:12.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:12 smithi022 ceph-mon[23758]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:12.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:12 smithi022 ceph-mon[28440]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:12.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:21:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:21:12] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.43.0" 2024-09-13T23:21:14.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:14 smithi076 ceph-mon[24587]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:14.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:14 smithi022 ceph-mon[28440]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:14.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:14 smithi022 ceph-mon[23758]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:16 smithi076 ceph-mon[24587]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:16.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:16 smithi022 ceph-mon[23758]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:16.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:16 smithi022 ceph-mon[28440]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:18 smithi076 ceph-mon[24587]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:18.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:18 smithi022 ceph-mon[23758]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:18.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:18 smithi022 ceph-mon[28440]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:19 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:19 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:19.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:19 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:20 smithi076 ceph-mon[24587]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:20 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:21:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:20 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:21:20.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:20 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:21:20.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:20 smithi022 ceph-mon[23758]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:20.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:20 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:21:20.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:20 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:21:20.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:20 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:21:20.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:20 smithi022 ceph-mon[28440]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:20.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:20 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:21:20.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:20 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:21:20.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:20 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' 2024-09-13T23:21:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:22 smithi076 ceph-mon[24587]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:22.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:22 smithi022 ceph-mon[23758]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:22.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:21:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:21:22] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.43.0" 2024-09-13T23:21:22.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:22 smithi022 ceph-mon[28440]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:24 smithi076 ceph-mon[24587]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:24.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:24 smithi022 ceph-mon[23758]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:24.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:24 smithi022 ceph-mon[28440]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:26 smithi076 ceph-mon[24587]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:26 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:21:26.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:26 smithi022 ceph-mon[23758]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:26 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:21:26.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:26 smithi022 ceph-mon[28440]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:26.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:26 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:21:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:28 smithi076 ceph-mon[24587]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:28.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:28 smithi022 ceph-mon[23758]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:28.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:28 smithi022 ceph-mon[28440]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:29.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:29 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:29.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:29 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:29.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:29 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:30 smithi076 ceph-mon[24587]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:30.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:30 smithi022 ceph-mon[23758]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:30.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:30 smithi022 ceph-mon[28440]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:32 smithi076 ceph-mon[24587]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:32.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:32 smithi022 ceph-mon[23758]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:32.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:21:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:21:32] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.43.0" 2024-09-13T23:21:32.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:32 smithi022 ceph-mon[28440]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:34.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:34 smithi022 ceph-mon[23758]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:34.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:34 smithi022 ceph-mon[28440]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:34.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:34 smithi076 ceph-mon[24587]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:36.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:36 smithi022 ceph-mon[23758]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:36.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:36 smithi022 ceph-mon[28440]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:36.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:36 smithi076 ceph-mon[24587]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:38 smithi076 ceph-mon[24587]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:38.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:38 smithi022 ceph-mon[23758]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:38.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:38 smithi022 ceph-mon[28440]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:39.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:39 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:39.328 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:39 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:39.328 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:39 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:40 smithi076 ceph-mon[24587]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:40.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:40 smithi022 ceph-mon[23758]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:40.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:40 smithi022 ceph-mon[28440]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:41.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:41 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:21:41.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:41 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:21:41.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:41 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:21:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:42 smithi076 ceph-mon[24587]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:42.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:42 smithi022 ceph-mon[23758]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:42.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:42 smithi022 ceph-mon[28440]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:42.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:21:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:21:42] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.43.0" 2024-09-13T23:21:44.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:44 smithi022 ceph-mon[23758]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:44.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:44 smithi022 ceph-mon[28440]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:44.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:44 smithi076 ceph-mon[24587]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:46.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:46 smithi022 ceph-mon[23758]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:46.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:46 smithi022 ceph-mon[28440]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:46.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:46 smithi076 ceph-mon[24587]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:48.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:48 smithi076 ceph-mon[24587]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:48 smithi022 ceph-mon[23758]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:48 smithi022 ceph-mon[28440]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:49.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:49 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:49.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:49 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:49.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:49 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:50.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:50 smithi076 ceph-mon[24587]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:50.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:50 smithi022 ceph-mon[23758]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:50.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:50 smithi022 ceph-mon[28440]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:52.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:52 smithi022 ceph-mon[23758]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:52.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:21:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:21:52] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.43.0" 2024-09-13T23:21:52.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:52 smithi022 ceph-mon[28440]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:52.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:52 smithi076 ceph-mon[24587]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:54.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:54 smithi022 ceph-mon[23758]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:54.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:54 smithi022 ceph-mon[28440]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:54.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:54 smithi076 ceph-mon[24587]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:56.589 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:56 smithi076 ceph-mon[24587]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:56.589 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:56 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:21:56.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:56 smithi022 ceph-mon[23758]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:56.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:56 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:21:56.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:56 smithi022 ceph-mon[28440]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:21:56.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:56 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:21:58.040 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:58 smithi022 ceph-mon[23758]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:58.040 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:58 smithi022 ceph-mon[28440]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:58 smithi076 ceph-mon[24587]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:21:58.595 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:21:59.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:21:59 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:59.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:21:59 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:21:59.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:21:59 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:22:00.266 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:00 smithi022 ceph-mon[28440]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:00.266 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:00 smithi022 ceph-mon[23758]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:00 smithi076 ceph-mon[24587]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:00.862 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:22:00.862 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (9m) 2m ago 17m 24.8M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:22:00.862 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (8m) 2m ago 16m 53.2M - 8.3.5 dad864ee21e9 969941fc6e23 2024-09-13T23:22:00.862 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (2m) 2m ago 16m 52.4M - 3.5 e1d6a67b021e 2a49abbe69cf 2024-09-13T23:22:00.862 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283 running (12m) 2m ago 21m 438M - 19.1.1-324-gecab7be5 0652ff2b5d78 194c1a9a8cc9 2024-09-13T23:22:00.862 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (5m) 2m ago 23m 491M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:22:00.862 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (23m) 2m ago 23m 62.9M 2048M 17.2.0 e1d6a67b021e 7d59bd9c71e8 2024-09-13T23:22:00.862 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (22m) 2m ago 22m 50.1M 2048M 17.2.0 e1d6a67b021e a1194b4b1c43 2024-09-13T23:22:00.862 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (22m) 2m ago 22m 47.1M 2048M 17.2.0 e1d6a67b021e 336a6f00a2a7 2024-09-13T23:22:00.862 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (9m) 2m ago 17m 15.7M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:22:00.863 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (8m) 2m ago 17m 15.9M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:22:00.863 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (21m) 2m ago 21m 52.3M 2993M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:22:00.863 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (21m) 2m ago 21m 54.0M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:22:00.863 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (20m) 2m ago 20m 50.0M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:22:00.863 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (20m) 2m ago 20m 51.2M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:22:00.863 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (19m) 2m ago 19m 52.9M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:22:00.863 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (19m) 2m ago 19m 54.8M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:22:00.863 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (19m) 2m ago 19m 53.7M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:22:00.863 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (18m) 2m ago 18m 53.0M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:22:00.863 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (2m) 2m ago 17m 44.0M - 2.43.0 a07b618ecd1d f26b8c02ef4e 2024-09-13T23:22:00.863 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (16m) 2m ago 16m 74.8M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:22:00.863 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (16m) 2m ago 16m 73.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:22:01.522 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T23:22:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:02 smithi022 ceph-mon[23758]: from='client.25240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:22:02.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:02 smithi022 ceph-mon[23758]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:02.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:02 smithi022 ceph-mon[28440]: from='client.25240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:22:02.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:02 smithi022 ceph-mon[28440]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:02.750 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:22:02] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.43.0" 2024-09-13T23:22:02.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:02 smithi076 ceph-mon[24587]: from='client.25240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:22:02.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:02 smithi076 ceph-mon[24587]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:03.669 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:22:03.669 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:22:03.669 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T23:22:03.669 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": {}, 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T23:22:03.670 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:22:03.671 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:22:03.671 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:22:04.297 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph -s' 2024-09-13T23:22:04.603 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:04 smithi022 ceph-mon[23758]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:04.603 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:04 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1109991171' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:22:04.604 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:04 smithi022 ceph-mon[28440]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:04.604 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:04 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1109991171' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:22:04.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:04 smithi076 ceph-mon[24587]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:04.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:04 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1109991171' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:22:06.402 INFO:teuthology.orchestra.run.smithi022.stdout: cluster: 2024-09-13T23:22:06.402 INFO:teuthology.orchestra.run.smithi022.stdout: id: 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:22:06.402 INFO:teuthology.orchestra.run.smithi022.stdout: health: HEALTH_OK 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: services: 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: mon: 3 daemons, quorum a,c,b (age 22m) 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: mgr: y(active, since 3m), standbys: x 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: data: 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: pools: 6 pools, 161 pgs 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: objects: 209 objects, 585 KiB 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: pgs: 161 active+clean 2024-09-13T23:22:06.403 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:22:06.404 INFO:teuthology.orchestra.run.smithi022.stdout: io: 2024-09-13T23:22:06.404 INFO:teuthology.orchestra.run.smithi022.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-13T23:22:06.404 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-13T23:22:06.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:06 smithi022 ceph-mon[23758]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:06.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:06 smithi022 ceph-mon[28440]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:06.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:06 smithi076 ceph-mon[24587]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:07.117 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T23:22:07.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:07 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3684847342' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T23:22:07.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:07 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3684847342' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T23:22:07.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:07 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3684847342' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T23:22:08.400 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:08 smithi022 ceph-mon[23758]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:08.400 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:08 smithi022 ceph-mon[28440]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:08.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:08 smithi076 ceph-mon[24587]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:09.178 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-09-13T23:22:09.533 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:09 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:22:09.533 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:09 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2075880967' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:22:09.533 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:09 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:22:09.533 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:09 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2075880967' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:22:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:09 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:22:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:09 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2075880967' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:22:09.890 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-09-13T23:22:10.448 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:10 smithi022 ceph-mon[23758]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:10.448 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:10 smithi022 ceph-mon[28440]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:10.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:10 smithi076 ceph-mon[24587]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:11.354 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:11 smithi022 ceph-mon[23758]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:22:11.355 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:11 smithi022 ceph-mon[28440]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:22:11.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:11 smithi076 ceph-mon[24587]: from='mgr.15225 172.21.15.22:0/1117731108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:22:11.910 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:22:12.505 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:12 smithi022 ceph-mon[23758]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:12.505 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:12 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/220466263' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:22:12.506 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:22:12] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.43.0" 2024-09-13T23:22:12.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:12 smithi022 ceph-mon[28440]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:12.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:12 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/220466263' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:22:12.530 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph mgr fail' 2024-09-13T23:22:12.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:12 smithi076 ceph-mon[24587]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:12.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:12 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/220466263' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:22:14.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:14 smithi022 ceph-mon[23758]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:14.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:14 smithi022 ceph-mon[28440]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:14.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:14 smithi076 ceph-mon[24587]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:15.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3012239161' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:22:15.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-mon[23758]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T23:22:15.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:15.326+0000 7fc90d95e640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T23:22:15.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ignoring --setuser ceph since I am not root 2024-09-13T23:22:15.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ignoring --setgroup ceph since I am not root 2024-09-13T23:22:15.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-mgr[61727]: -- 172.21.15.22:0/3347325863 <== mon.1 v2:172.21.15.22:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555c89cbb0e0 con 0x555c89c99400 2024-09-13T23:22:15.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:15.488+0000 7f7ed50b9040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T23:22:15.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:15.548+0000 7f7ed50b9040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T23:22:15.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3012239161' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:22:15.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-mon[28440]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T23:22:15.694 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:15 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3012239161' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T23:22:15.694 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:15 smithi076 ceph-mon[24587]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T23:22:15.695 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:22:15] ENGINE Bus STOPPING 2024-09-13T23:22:15.695 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:22:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T23:22:15.695 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:22:15] ENGINE Bus STOPPED 2024-09-13T23:22:15.994 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:15.705+0000 7f7ed50b9040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T23:22:15.994 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:15.777+0000 7f7ed50b9040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T23:22:15.995 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:15.933+0000 7f7ed50b9040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:22:16.020 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 180' 2024-09-13T23:22:16.034 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:22:15] ENGINE Bus STARTING 2024-09-13T23:22:16.034 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:22:15] ENGINE Serving on http://:::9283 2024-09-13T23:22:16.035 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:22:15] ENGINE Bus STARTED 2024-09-13T23:22:16.323 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:16.087+0000 7f7ed50b9040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T23:22:16.649 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:16.442+0000 7f7ed50b9040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T23:22:16.649 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:16.506+0000 7f7ed50b9040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T23:22:16.649 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3012239161' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T23:22:16.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:22:16.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:22:16.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:22:16.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: mgrmap e32: x(active, starting, since 0.789039s) 2024-09-13T23:22:16.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:22:16.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:22:16.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:22:16.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:22:16.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:22:16.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:22:16.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:22:16.650 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:22:16.651 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:22:16.651 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:22:16.651 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:22:16.651 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:22:16.651 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: Manager daemon x is now available 2024-09-13T23:22:16.651 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:22:16.651 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:22:16.651 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:22:16.651 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T23:22:16.651 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T23:22:16.652 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3012239161' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T23:22:16.652 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:22:16.652 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:22:16.652 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:22:16.652 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: mgrmap e32: x(active, starting, since 0.789039s) 2024-09-13T23:22:16.652 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:22:16.652 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:22:16.652 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:22:16.652 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:22:16.652 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:22:16.652 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:22:16.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:22:16.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:22:16.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:22:16.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:22:16.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:22:16.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:22:16.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: Manager daemon x is now available 2024-09-13T23:22:16.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:22:16.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:22:16.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:22:16.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T23:22:16.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T23:22:16.703 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3012239161' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T23:22:16.703 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:22:16.703 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:22:16.703 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:22:16.703 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: mgrmap e32: x(active, starting, since 0.789039s) 2024-09-13T23:22:16.703 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:22:16.703 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:22:16.703 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:22:16.703 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:22:16.703 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:22:16.704 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:22:16.704 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:22:16.704 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:22:16.704 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:22:16.704 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:22:16.704 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:22:16.704 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:22:16.704 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: Manager daemon x is now available 2024-09-13T23:22:16.704 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:22:16.704 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:22:16.704 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:22:16.704 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T23:22:16.705 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:16 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T23:22:16.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:16 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:16.717+0000 7f7ed50b9040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T23:22:17.647 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:17 smithi022 ceph-mon[28440]: mgrmap e33: x(active, since 2s) 2024-09-13T23:22:17.647 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:17 smithi022 ceph-mon[28440]: [13/Sep/2024:23:22:17] ENGINE Bus STARTING 2024-09-13T23:22:17.647 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:17 smithi022 ceph-mon[23758]: mgrmap e33: x(active, since 2s) 2024-09-13T23:22:17.647 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:17 smithi022 ceph-mon[23758]: [13/Sep/2024:23:22:17] ENGINE Bus STARTING 2024-09-13T23:22:17.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-mon[24587]: mgrmap e33: x(active, since 2s) 2024-09-13T23:22:17.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-mon[24587]: [13/Sep/2024:23:22:17] ENGINE Bus STARTING 2024-09-13T23:22:17.788 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:22:17.450+0000 7f5024737640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:23:22:17] ENGINE Error in HTTPServer.serve 2024-09-13T23:22:17.788 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:22:17.788 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:22:17.788 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: self._connections.run(self.expiration_interval) 2024-09-13T23:22:17.788 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:22:17.788 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: self._run(expiration_interval) 2024-09-13T23:22:17.788 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:22:17.789 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:22:17.789 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:22:17.789 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:22:17.789 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:22:17.789 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: s = self.context.wrap_socket( 2024-09-13T23:22:17.789 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:22:17.789 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.sslsocket_class._create( 2024-09-13T23:22:17.789 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:22:17.789 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: self.do_handshake() 2024-09-13T23:22:17.789 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:22:17.789 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: self._sslobj.do_handshake() 2024-09-13T23:22:17.790 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:22:17.790 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:17 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:22:17.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:17 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:17.671+0000 7f7ed50b9040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T23:22:17.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:17 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:17.744+0000 7f7ed50b9040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T23:22:18.641 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:18.515+0000 7f7ed50b9040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T23:22:18.641 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-mon[23758]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:18.641 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-mon[23758]: [13/Sep/2024:23:22:17] ENGINE Serving on http://172.21.15.76:8765 2024-09-13T23:22:18.641 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-mon[23758]: [13/Sep/2024:23:22:17] ENGINE Serving on https://172.21.15.76:7150 2024-09-13T23:22:18.641 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-mon[23758]: [13/Sep/2024:23:22:17] ENGINE Bus STARTED 2024-09-13T23:22:18.641 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-mon[23758]: [13/Sep/2024:23:22:17] ENGINE Error in HTTPServer.serve 2024-09-13T23:22:18.641 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:22:18.642 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:22:18.642 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:22:18.642 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:22:18.642 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._run(expiration_interval) 2024-09-13T23:22:18.642 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:22:18.642 INFO:journalctl@ceph.mon.a.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:22:18.642 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:22:18.642 INFO:journalctl@ceph.mon.a.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:22:18.642 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:22:18.642 INFO:journalctl@ceph.mon.a.smithi022.stdout: s = self.context.wrap_socket( 2024-09-13T23:22:18.642 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:22:18.642 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.sslsocket_class._create( 2024-09-13T23:22:18.643 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:22:18.643 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.do_handshake() 2024-09-13T23:22:18.643 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:22:18.643 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._sslobj.do_handshake() 2024-09-13T23:22:18.643 INFO:journalctl@ceph.mon.a.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:22:18.643 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-mon[23758]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:22:18.643 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-mon[28440]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:18.643 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-mon[28440]: [13/Sep/2024:23:22:17] ENGINE Serving on http://172.21.15.76:8765 2024-09-13T23:22:18.643 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-mon[28440]: [13/Sep/2024:23:22:17] ENGINE Serving on https://172.21.15.76:7150 2024-09-13T23:22:18.643 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-mon[28440]: [13/Sep/2024:23:22:17] ENGINE Bus STARTED 2024-09-13T23:22:18.643 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-mon[28440]: [13/Sep/2024:23:22:17] ENGINE Error in HTTPServer.serve 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._run(expiration_interval) 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: s = self.context.wrap_socket( 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:22:18.644 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.sslsocket_class._create( 2024-09-13T23:22:18.645 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:22:18.645 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.do_handshake() 2024-09-13T23:22:18.645 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:22:18.645 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._sslobj.do_handshake() 2024-09-13T23:22:18.645 INFO:journalctl@ceph.mon.c.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:22:18.645 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-mon[28440]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:22:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:18 smithi076 ceph-mon[24587]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:18 smithi076 ceph-mon[24587]: [13/Sep/2024:23:22:17] ENGINE Serving on http://172.21.15.76:8765 2024-09-13T23:22:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:18 smithi076 ceph-mon[24587]: [13/Sep/2024:23:22:17] ENGINE Serving on https://172.21.15.76:7150 2024-09-13T23:22:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:18 smithi076 ceph-mon[24587]: [13/Sep/2024:23:22:17] ENGINE Bus STARTED 2024-09-13T23:22:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:18 smithi076 ceph-mon[24587]: [13/Sep/2024:23:22:17] ENGINE Error in HTTPServer.serve 2024-09-13T23:22:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:22:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:22:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:22:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:22:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._run(expiration_interval) 2024-09-13T23:22:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:22:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:22:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:22:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:22:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:22:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout: s = self.context.wrap_socket( 2024-09-13T23:22:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:22:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.sslsocket_class._create( 2024-09-13T23:22:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:22:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout: self.do_handshake() 2024-09-13T23:22:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:22:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._sslobj.do_handshake() 2024-09-13T23:22:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:22:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:18 smithi076 ceph-mon[24587]: from='client.15324 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:22:18.919 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:18.640+0000 7f7ed50b9040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T23:22:18.919 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:18.801+0000 7f7ed50b9040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:22:18.919 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:18 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:18.866+0000 7f7ed50b9040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T23:22:19.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:19.028+0000 7f7ed50b9040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T23:22:19.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:19.103+0000 7f7ed50b9040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T23:22:19.177 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:19.167+0000 7f7ed50b9040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T23:22:19.507 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:19.357+0000 7f7ed50b9040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T23:22:19.507 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:19.431+0000 7f7ed50b9040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T23:22:19.507 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:19.506+0000 7f7ed50b9040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T23:22:19.850 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:19 smithi076 ceph-mon[24587]: mgrmap e34: x(active, since 4s) 2024-09-13T23:22:19.857 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:19 smithi022 ceph-mon[23758]: mgrmap e34: x(active, since 4s) 2024-09-13T23:22:19.857 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:19 smithi022 ceph-mon[28440]: mgrmap e34: x(active, since 4s) 2024-09-13T23:22:19.857 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:19.603+0000 7f7ed50b9040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T23:22:19.857 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:19.670+0000 7f7ed50b9040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T23:22:19.858 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:19.733+0000 7f7ed50b9040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T23:22:20.869 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:20 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:20.595+0000 7f7ed50b9040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T23:22:20.869 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:20 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:22:20] ENGINE Bus STARTING 2024-09-13T23:22:20.869 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:20 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: CherryPy Checker: 2024-09-13T23:22:20.869 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:20 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: The Application mounted at '' has an empty config. 2024-09-13T23:22:20.870 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:20 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:22:20.870 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:20 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:22:20] ENGINE Serving on http://:::9283 2024-09-13T23:22:20.870 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:20 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:22:20] ENGINE Bus STARTED 2024-09-13T23:22:21.511 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[23758]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:21.511 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:21.512 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:21.512 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[23758]: Standby manager daemon y started 2024-09-13T23:22:21.512 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T23:22:21.512 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:22:21.512 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T23:22:21.512 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:22:21.512 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:21.512 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:21.512 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[28440]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:21.512 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:21.513 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:21.513 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[28440]: Standby manager daemon y started 2024-09-13T23:22:21.513 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T23:22:21.513 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:22:21.513 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T23:22:21.513 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:22:21.513 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:21.513 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:21 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:21 smithi076 ceph-mon[24587]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:21 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:21 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:21.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:21 smithi076 ceph-mon[24587]: Standby manager daemon y started 2024-09-13T23:22:21.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:21 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T23:22:21.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:21 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:22:21.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:21 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T23:22:21.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:21 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:22:21.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:21 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:21.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:21 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:22.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:22 smithi022 ceph-mon[23758]: mgrmap e35: x(active, since 6s), standbys: y 2024-09-13T23:22:22.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:22 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:22:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:22 smithi022 ceph-mon[28440]: mgrmap e35: x(active, since 6s), standbys: y 2024-09-13T23:22:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:22 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:22:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:22 smithi076 ceph-mon[24587]: mgrmap e35: x(active, since 6s), standbys: y 2024-09-13T23:22:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:22 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:22:22.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:22:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:22:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T23:22:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:23 smithi076 ceph-mon[24587]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:23 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:23 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:23 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:23 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:23 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:23 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:22:23.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[23758]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:23.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:23.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:23.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:23.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:23.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:23.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:22:23.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[28440]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:23.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:23.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:23.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:23.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:23.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:23.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:23 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:22:24.517 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[23758]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:22:24.517 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[23758]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:22:24.517 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.517 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.517 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.517 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.517 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.518 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:22:24.518 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:22:24.518 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:24.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:22:24.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:22:24.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.518 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:22:24.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:22:24.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:24 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:24 smithi076 ceph-mon[24587]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:22:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:24 smithi076 ceph-mon[24587]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:22:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:24 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:24 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:24 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:24 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:24 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:24 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:22:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:24 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:22:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:24 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[28440]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T23:22:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[28440]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:22:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[28440]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:22:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:22:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:22:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[28440]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:22:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[28440]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:22:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[28440]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:22:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[28440]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:22:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[23758]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T23:22:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[23758]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:22:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[23758]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:22:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[23758]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:22:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[23758]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:22:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[23758]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:22:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[23758]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:22:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[23758]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:22:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:25 smithi022 ceph-mon[23758]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:22:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:25 smithi076 ceph-mon[24587]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T23:22:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:25 smithi076 ceph-mon[24587]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:22:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:25 smithi076 ceph-mon[24587]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:22:25.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:25 smithi076 ceph-mon[24587]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:22:25.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:25 smithi076 ceph-mon[24587]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:22:25.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:25 smithi076 ceph-mon[24587]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:22:25.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:25 smithi076 ceph-mon[24587]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:22:25.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:25 smithi076 ceph-mon[24587]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:22:25.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:25 smithi076 ceph-mon[24587]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:22:27.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:27 smithi022 ceph-mon[23758]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T23:22:27.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:27 smithi022 ceph-mon[28440]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T23:22:27.502 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-mon[24587]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T23:22:27.503 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:22:27.254+0000 7f505d9e6640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:22:27.503 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Creating ceph-iscsi config... 2024-09-13T23:22:27.503 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:22:27.503 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:22:27.503 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:22:27.503 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:27.503 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:27.504 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:22:27.504 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:22:27.504 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return _run_code(code, main_globals, None, 2024-09-13T23:22:27.504 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:22:27.504 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: exec(code, run_globals) 2024-09-13T23:22:27.504 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:22:27.504 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:22:27.504 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:22:27.504 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:22:27.504 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:22:27.504 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:22:27.504 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:22:27.505 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:27.505 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:27.505 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:22:27.505 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:22:27.505 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:22:27.505 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:22:27.505 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.wait_async( 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return future.result(timeout) 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return self.__get_result() 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: raise self._exception 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: out, err, code = await self._run_cephadm( 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: raise OrchestratorError( 2024-09-13T23:22:27.506 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Creating ceph-iscsi config... 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: Traceback (most recent call last): 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: return _run_code(code, main_globals, None, 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: exec(code, run_globals) 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:22:27.507 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:22:27.508 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:22:27.508 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:22:27.508 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:22:27.508 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:22:27.508 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:22:27.508 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:27.508 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:28 smithi076 ceph-mon[24587]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:22:28.537 INFO:journalctl@ceph.mon.b.smithi076.stdout: Creating ceph-iscsi config... 2024-09-13T23:22:28.537 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:22:28.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:22:28.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:22:28.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:28.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:22:28.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:22:28.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:22:28.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:22:28.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: exec(code, run_globals) 2024-09-13T23:22:28.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:22:28.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.wait_async( 2024-09-13T23:22:28.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:22:28.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:22:28.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:22:28.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: return future.result(timeout) 2024-09-13T23:22:28.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:22:28.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.__get_result() 2024-09-13T23:22:28.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:22:28.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: raise self._exception 2024-09-13T23:22:28.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:22:28.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: out, err, code = await self._run_cephadm( 2024-09-13T23:22:28.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:22:28.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: raise OrchestratorError( 2024-09-13T23:22:28.540 INFO:journalctl@ceph.mon.b.smithi076.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: Creating ceph-iscsi config... 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: exec(code, run_globals) 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:22:28.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:22:28.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:22:28.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:22:28.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:22:28.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:22:28.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:22:28.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:28.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:28 smithi076 ceph-mon[24587]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:22:28.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:28 smithi022 ceph-mon[23758]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:22:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:22:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:22:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:22:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:22:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:22:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:22:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:22:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:22:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:22:28.678 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.wait_async( 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: return future.result(timeout) 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.__get_result() 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: raise self._exception 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: out, err, code = await self._run_cephadm( 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: raise OrchestratorError( 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:22:28.679 INFO:journalctl@ceph.mon.a.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:22:28.680 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:22:28.681 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:22:28.681 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:22:28.681 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:22:28.681 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:22:28.681 INFO:journalctl@ceph.mon.a.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:28.681 INFO:journalctl@ceph.mon.a.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:28 smithi022 ceph-mon[23758]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:22:28.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:28 smithi022 ceph-mon[28440]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:22:28.681 INFO:journalctl@ceph.mon.c.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:22:28.681 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:22:28.681 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:22:28.681 INFO:journalctl@ceph.mon.c.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:22:28.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:28.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:22:28.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:22:28.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:22:28.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:22:28.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:22:28.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:22:28.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:22:28.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:22:28.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:22:28.682 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.wait_async( 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T23:22:28.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: return future.result(timeout) 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.__get_result() 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: raise self._exception 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: out, err, code = await self._run_cephadm( 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: raise OrchestratorError( 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: Creating ceph-iscsi config... 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:22:28.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: Non-zero exit code 1 from systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T23:22:28.685 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:22:28.686 INFO:journalctl@ceph.mon.c.smithi022.stdout: RuntimeError: Failed command: systemctl restart ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:22:28.686 INFO:journalctl@ceph.mon.c.smithi022.stdout: See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:22:28.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:28 smithi022 ceph-mon[28440]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:22:29.330 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:29 smithi076 ceph-mon[24587]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T23:22:29.330 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:29 smithi076 ceph-mon[24587]: Reconfiguring daemon prometheus.a on smithi076 2024-09-13T23:22:29.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:29 smithi022 ceph-mon[23758]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T23:22:29.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:29 smithi022 ceph-mon[23758]: Reconfiguring daemon prometheus.a on smithi076 2024-09-13T23:22:29.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:29 smithi022 ceph-mon[28440]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T23:22:29.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:29 smithi022 ceph-mon[28440]: Reconfiguring daemon prometheus.a on smithi076 2024-09-13T23:22:30.768 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 systemd[1]: Stopping Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:22:31.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:22:30.767Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T23:22:31.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:22:30.767Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T23:22:31.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:22:30.767Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T23:22:31.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:22:30.767Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T23:22:31.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:22:30.767Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T23:22:31.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:22:30.767Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T23:22:31.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:22:30.767Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T23:22:31.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:22:30.767Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T23:22:31.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:22:30.767Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T23:22:31.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:22:30.768Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T23:22:31.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:22:30.768Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T23:22:31.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[53595]: ts=2024-09-13T23:22:30.768Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T23:22:31.038 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:30 smithi076 podman[56544]: 2024-09-13 23:22:30.777305207 +0000 UTC m=+0.128385214 container died f26b8c02ef4e78ffd7223403d746a6fa4e46add2db60e56e25e953fba5240036 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:22:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:31 smithi076 ceph-mon[24587]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:22:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:31 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:22:31.537 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:31 smithi076 podman[56544]: 2024-09-13 23:22:31.100126643 +0000 UTC m=+0.451206641 container cleanup f26b8c02ef4e78ffd7223403d746a6fa4e46add2db60e56e25e953fba5240036 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:22:31.537 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:31 smithi076 bash[56544]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T23:22:31.538 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:31 smithi076 podman[56556]: 2024-09-13 23:22:31.326400565 +0000 UTC m=+0.549292435 container remove f26b8c02ef4e78ffd7223403d746a6fa4e46add2db60e56e25e953fba5240036 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:22:31.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:31 smithi022 ceph-mon[23758]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:22:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:31 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:22:31.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:31 smithi022 ceph-mon[28440]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:22:31.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:31 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:22:31.932 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:31 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T23:22:31.933 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:31 smithi076 systemd[1]: Stopped Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:22:31.933 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:31 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.730s CPU time. 2024-09-13T23:22:31.933 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:31 smithi076 systemd[1]: Starting Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:22:31.933 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:31 smithi076 podman[56651]: 2024-09-13 23:22:31.687632598 +0000 UTC m=+0.017081593 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T23:22:31.933 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:31 smithi076 podman[56651]: 2024-09-13 23:22:31.810780166 +0000 UTC m=+0.140229155 container create 959260f15170e55f4c342c455bd3eeeac670987d4cd2a6a01816c038b23490a4 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:22:32.287 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 podman[56651]: 2024-09-13 23:22:32.047123693 +0000 UTC m=+0.376572688 container init 959260f15170e55f4c342c455bd3eeeac670987d4cd2a6a01816c038b23490a4 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:22:32.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 podman[56651]: 2024-09-13 23:22:32.050060505 +0000 UTC m=+0.379509497 container start 959260f15170e55f4c342c455bd3eeeac670987d4cd2a6a01816c038b23490a4 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:22:32.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.073Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T23:22:32.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.073Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T23:22:32.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.073Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi076 (none))" 2024-09-13T23:22:32.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.073Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T23:22:32.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.073Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T23:22:32.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.075Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T23:22:32.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.075Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T23:22:32.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.076Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T23:22:32.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.076Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T23:22:32.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.077Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T23:22:32.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.077Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.553µs 2024-09-13T23:22:32.289 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.077Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T23:22:32.289 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.084Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-09-13T23:22:32.289 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.099Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-09-13T23:22:32.289 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.109Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-09-13T23:22:32.289 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.111Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-09-13T23:22:32.289 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.111Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-09-13T23:22:32.289 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.111Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=19.419µs wal_replay_duration=33.747921ms wbl_replay_duration=115ns total_replay_duration=33.785197ms 2024-09-13T23:22:32.289 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.114Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T23:22:32.289 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.114Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T23:22:32.289 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.114Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T23:22:32.289 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.134Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.664319ms db_storage=734ns remote_storage=1.06µs web_handler=435ns query_engine=610ns scrape=408.334µs scrape_sd=94.307µs notify=9.587µs notify_sd=9.755µs rules=18.816468ms tracing=5.351µs 2024-09-13T23:22:32.290 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.134Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T23:22:32.290 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:22:32.134Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T23:22:32.290 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 bash[56651]: 959260f15170e55f4c342c455bd3eeeac670987d4cd2a6a01816c038b23490a4 2024-09-13T23:22:32.290 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:22:32 smithi076 systemd[1]: Started Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:22:33.503 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:33 smithi076 ceph-mon[24587]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:22:33.503 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:33 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:33.503 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:33 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:33.503 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:33 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:22:33.504 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:33 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:22:33.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:33 smithi022 ceph-mon[28440]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:22:33.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:33 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:33.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:33 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:33.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:33 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:22:33.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:33 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:22:33.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:33 smithi022 ceph-mon[23758]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:22:33.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:33 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:33.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:33 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:33.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:33 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:22:33.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:33 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:22:35.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:35 smithi076 ceph-mon[24587]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:22:35.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:35 smithi022 ceph-mon[23758]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:22:35.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:35 smithi022 ceph-mon[28440]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:22:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:37 smithi076 ceph-mon[24587]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:22:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:37 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:37 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:37 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:37 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:22:37.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:37 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:37.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:37 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:22:37.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:37 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:22:37.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:37 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[28440]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:22:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:22:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:22:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:22:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:37.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[23758]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:22:37.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:37.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:37.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:37.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:22:37.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:37.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:22:37.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:22:37.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:37 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:38.391 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:38 smithi022 ceph-mon[23758]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:22:38.391 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:38 smithi022 ceph-mon[23758]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:22:38.391 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:38 smithi022 ceph-mon[28440]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:22:38.391 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:38 smithi022 ceph-mon[28440]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:22:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:38 smithi076 ceph-mon[24587]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:22:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:38 smithi076 ceph-mon[24587]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:22:39.410 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:39 smithi022 ceph-mon[23758]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:22:39.411 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:39 smithi022 ceph-mon[28440]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:22:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:39 smithi076 ceph-mon[24587]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T23:22:41.313 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:41 smithi022 ceph-mon[23758]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:41.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:41 smithi022 ceph-mon[28440]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:41 smithi076 ceph-mon[24587]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:42.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:22:41] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.43.0" 2024-09-13T23:22:42.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:42 smithi076 ceph-mon[24587]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:42.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:42 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:42.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:42 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:42.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:42 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:22:42.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:42 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:22:42.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:42 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:22:42.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:42 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:22:42.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:42 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3138391393' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:22:42.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:42 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/254043516' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/254935170"}]: dispatch 2024-09-13T23:22:42.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:42 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/254935170"}]: dispatch 2024-09-13T23:22:42.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[28440]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:42.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:42.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:42.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:22:42.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:22:42.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:22:42.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:22:42.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3138391393' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:22:42.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/254043516' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/254935170"}]: dispatch 2024-09-13T23:22:42.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/254935170"}]: dispatch 2024-09-13T23:22:42.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[23758]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:22:42.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:42.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:42.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:22:42.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:22:42.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:22:42.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:22:42.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3138391393' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:22:42.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/254043516' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/254935170"}]: dispatch 2024-09-13T23:22:42.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:42 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/254935170"}]: dispatch 2024-09-13T23:22:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:44 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/254935170"}]': finished 2024-09-13T23:22:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:44 smithi076 ceph-mon[24587]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T23:22:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:44 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3640443140' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/2362433894"}]: dispatch 2024-09-13T23:22:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:44 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/2362433894"}]: dispatch 2024-09-13T23:22:44.417 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:44 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/254935170"}]': finished 2024-09-13T23:22:44.417 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:44 smithi022 ceph-mon[28440]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T23:22:44.417 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:44 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3640443140' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/2362433894"}]: dispatch 2024-09-13T23:22:44.417 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:44 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/2362433894"}]: dispatch 2024-09-13T23:22:44.418 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:44 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/254935170"}]': finished 2024-09-13T23:22:44.418 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:44 smithi022 ceph-mon[23758]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T23:22:44.418 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:44 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3640443140' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/2362433894"}]: dispatch 2024-09-13T23:22:44.418 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:44 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/2362433894"}]: dispatch 2024-09-13T23:22:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:45 smithi076 ceph-mon[24587]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-13T23:22:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:45 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/2362433894"}]': finished 2024-09-13T23:22:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:45 smithi076 ceph-mon[24587]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T23:22:45.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:45 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1745960536' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/2362433894"}]: dispatch 2024-09-13T23:22:45.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:45 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/2362433894"}]: dispatch 2024-09-13T23:22:45.303 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:45 smithi022 ceph-mon[23758]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-13T23:22:45.303 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:45 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/2362433894"}]': finished 2024-09-13T23:22:45.304 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:45 smithi022 ceph-mon[23758]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T23:22:45.304 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:45 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1745960536' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/2362433894"}]: dispatch 2024-09-13T23:22:45.304 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:45 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/2362433894"}]: dispatch 2024-09-13T23:22:45.304 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:45 smithi022 ceph-mon[28440]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-13T23:22:45.304 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:45 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/2362433894"}]': finished 2024-09-13T23:22:45.304 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:45 smithi022 ceph-mon[28440]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T23:22:45.304 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:45 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1745960536' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/2362433894"}]: dispatch 2024-09-13T23:22:45.304 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:45 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/2362433894"}]: dispatch 2024-09-13T23:22:46.050 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:46 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/2362433894"}]': finished 2024-09-13T23:22:46.050 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:46 smithi022 ceph-mon[28440]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T23:22:46.050 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/478225673' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2577026609"}]: dispatch 2024-09-13T23:22:46.051 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:46 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:22:46.051 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:46 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/2362433894"}]': finished 2024-09-13T23:22:46.051 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:46 smithi022 ceph-mon[23758]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T23:22:46.051 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:46 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/478225673' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2577026609"}]: dispatch 2024-09-13T23:22:46.051 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:46 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:22:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:46 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/2362433894"}]': finished 2024-09-13T23:22:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:46 smithi076 ceph-mon[24587]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T23:22:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:46 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/478225673' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2577026609"}]: dispatch 2024-09-13T23:22:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:46 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:22:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:47 smithi076 ceph-mon[24587]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T23:22:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:47 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/478225673' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2577026609"}]': finished 2024-09-13T23:22:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:47 smithi076 ceph-mon[24587]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T23:22:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:47 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:47 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:47 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:47.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:47 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:22:47.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:47 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:47.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:47 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3139967903' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3306975790"}]: dispatch 2024-09-13T23:22:47.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:47 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3306975790"}]: dispatch 2024-09-13T23:22:47.330 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[28440]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T23:22:47.330 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/478225673' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2577026609"}]': finished 2024-09-13T23:22:47.330 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[28440]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T23:22:47.330 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:47.331 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:47.331 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:47.331 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:22:47.331 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:47.331 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3139967903' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3306975790"}]: dispatch 2024-09-13T23:22:47.331 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3306975790"}]: dispatch 2024-09-13T23:22:47.331 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[23758]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T23:22:47.331 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/478225673' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/2577026609"}]': finished 2024-09-13T23:22:47.331 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[23758]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T23:22:47.331 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:47.331 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:47.332 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:22:47.332 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:22:47.332 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:22:47.332 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3139967903' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3306975790"}]: dispatch 2024-09-13T23:22:47.332 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:47 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3306975790"}]: dispatch 2024-09-13T23:22:48.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:48 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3306975790"}]': finished 2024-09-13T23:22:48.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:48 smithi022 ceph-mon[28440]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T23:22:48.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:48 smithi022 ceph-mon[28440]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:48.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:48 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2093688063' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1387100505"}]: dispatch 2024-09-13T23:22:48.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:48 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1387100505"}]: dispatch 2024-09-13T23:22:48.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:48 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3306975790"}]': finished 2024-09-13T23:22:48.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:48 smithi022 ceph-mon[23758]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T23:22:48.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:48 smithi022 ceph-mon[23758]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:48.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:48 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2093688063' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1387100505"}]: dispatch 2024-09-13T23:22:48.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:48 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1387100505"}]: dispatch 2024-09-13T23:22:48.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:48 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3306975790"}]': finished 2024-09-13T23:22:48.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:48 smithi076 ceph-mon[24587]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T23:22:48.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:48 smithi076 ceph-mon[24587]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:48.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:48 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2093688063' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1387100505"}]: dispatch 2024-09-13T23:22:48.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:48 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1387100505"}]: dispatch 2024-09-13T23:22:49.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:49 smithi022 ceph-mon[28440]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1387100505"}]': finished 2024-09-13T23:22:49.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:49 smithi022 ceph-mon[28440]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T23:22:49.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:49 smithi022 ceph-mon[23758]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1387100505"}]': finished 2024-09-13T23:22:49.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:49 smithi022 ceph-mon[23758]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T23:22:49.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:49 smithi076 ceph-mon[24587]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/1387100505"}]': finished 2024-09-13T23:22:49.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:49 smithi076 ceph-mon[24587]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T23:22:50.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:50 smithi022 ceph-mon[23758]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:50.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:50 smithi022 ceph-mon[28440]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:50.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:50 smithi076 ceph-mon[24587]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:22:52.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:22:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:22:51] "GET /metrics HTTP/1.1" 200 37892 "" "Prometheus/2.43.0" 2024-09-13T23:22:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:53 smithi076 ceph-mon[24587]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:53 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:22:53.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:53 smithi022 ceph-mon[23758]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:53.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:53 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:22:53.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:53 smithi022 ceph-mon[28440]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:22:53.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:53 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:22:55.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:55 smithi076 ceph-mon[24587]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T23:22:55.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:55 smithi022 ceph-mon[23758]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T23:22:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:55 smithi022 ceph-mon[28440]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T23:22:57.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:57 smithi076 ceph-mon[24587]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 637 B/s rd, 0 op/s 2024-09-13T23:22:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:57 smithi022 ceph-mon[23758]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 637 B/s rd, 0 op/s 2024-09-13T23:22:57.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:57 smithi022 ceph-mon[28440]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 637 B/s rd, 0 op/s 2024-09-13T23:22:59.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:22:59 smithi076 ceph-mon[24587]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:22:59.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:22:59 smithi022 ceph-mon[23758]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:22:59.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:22:59 smithi022 ceph-mon[28440]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:23:01.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:01 smithi076 ceph-mon[24587]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 928 B/s rd, 0 op/s 2024-09-13T23:23:01.333 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:01 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:23:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:01 smithi022 ceph-mon[23758]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 928 B/s rd, 0 op/s 2024-09-13T23:23:01.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:01 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:23:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:01 smithi022 ceph-mon[28440]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 928 B/s rd, 0 op/s 2024-09-13T23:23:01.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:01 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:23:02.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:23:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:23:01] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-13T23:23:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:03 smithi076 ceph-mon[24587]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:03 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:03.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:03 smithi022 ceph-mon[23758]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:03.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:03 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:03.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:03 smithi022 ceph-mon[28440]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:03.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:03 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:05 smithi076 ceph-mon[24587]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:05.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:05 smithi022 ceph-mon[23758]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:05.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:05 smithi022 ceph-mon[28440]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:07.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:07 smithi022 ceph-mon[23758]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:07 smithi022 ceph-mon[28440]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:07 smithi076 ceph-mon[24587]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:09.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:09 smithi022 ceph-mon[28440]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:09.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:09 smithi022 ceph-mon[23758]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:09 smithi076 ceph-mon[24587]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:11.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:11 smithi022 ceph-mon[23758]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:11.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:11 smithi022 ceph-mon[28440]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:11 smithi076 ceph-mon[24587]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:12.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:23:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:23:11] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-13T23:23:13.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:13 smithi022 ceph-mon[28440]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:13.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:13 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:13.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:13 smithi022 ceph-mon[23758]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:13.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:13 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:13.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:13 smithi076 ceph-mon[24587]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:13.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:13 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:15 smithi022 ceph-mon[28440]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:15.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:15 smithi022 ceph-mon[23758]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:15.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:15 smithi076 ceph-mon[24587]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:16.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:23:16.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:23:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:23:17.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:17 smithi022 ceph-mon[23758]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:17 smithi022 ceph-mon[28440]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:17.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:17 smithi076 ceph-mon[24587]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:19 smithi022 ceph-mon[23758]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:19 smithi022 ceph-mon[28440]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:19 smithi076 ceph-mon[24587]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:21.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:21 smithi022 ceph-mon[23758]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:21.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:21 smithi022 ceph-mon[28440]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:21 smithi076 ceph-mon[24587]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:22.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:23:21 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:23:21] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-13T23:23:23.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:23 smithi022 ceph-mon[28440]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:23.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:23 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:23.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:23 smithi022 ceph-mon[23758]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:23.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:23 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:23 smithi076 ceph-mon[24587]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:23 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:25.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:25 smithi022 ceph-mon[28440]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:25.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:25 smithi022 ceph-mon[23758]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:25 smithi076 ceph-mon[24587]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:27.338 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:27 smithi076 ceph-mon[24587]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:27.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:27 smithi022 ceph-mon[28440]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:27.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:27 smithi022 ceph-mon[23758]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:29.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:29 smithi022 ceph-mon[28440]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:29.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:29 smithi022 ceph-mon[23758]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:29 smithi076 ceph-mon[24587]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:31.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:31 smithi022 ceph-mon[23758]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:23:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:31 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:23:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:31 smithi022 ceph-mon[28440]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:23:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:31 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:23:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:31 smithi076 ceph-mon[24587]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:23:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:31 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:23:32.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:23:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:23:31] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-13T23:23:33.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:33 smithi022 ceph-mon[23758]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:23:33.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:33 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:33.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:33 smithi022 ceph-mon[28440]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:23:33.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:33 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:33.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:33 smithi076 ceph-mon[24587]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:23:33.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:33 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:35.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:35 smithi022 ceph-mon[23758]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:35.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:35 smithi022 ceph-mon[28440]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:35.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:35 smithi076 ceph-mon[24587]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:37.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:37 smithi022 ceph-mon[23758]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:23:37.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:37 smithi022 ceph-mon[28440]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:23:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:37 smithi076 ceph-mon[24587]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:23:39.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:39 smithi022 ceph-mon[28440]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:39.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:39 smithi022 ceph-mon[23758]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:39 smithi076 ceph-mon[24587]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:41.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:41 smithi022 ceph-mon[23758]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:41.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:41 smithi022 ceph-mon[28440]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:41 smithi076 ceph-mon[24587]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:42.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:23:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:23:41] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-13T23:23:43.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:43 smithi022 ceph-mon[23758]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:43.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:43 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:43.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:43 smithi022 ceph-mon[28440]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:43.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:43 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:43 smithi076 ceph-mon[24587]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:43 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:45.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:45 smithi022 ceph-mon[23758]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:45.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:45 smithi022 ceph-mon[28440]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:45 smithi076 ceph-mon[24587]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:46.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:46 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:23:46.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:46 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:23:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:46 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:23:47.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:47 smithi022 ceph-mon[23758]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:47.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:47 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:23:47.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:47 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:23:47.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:47 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:23:47.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:47 smithi022 ceph-mon[28440]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:47.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:47 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:23:47.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:47 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:23:47.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:47 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:23:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:47 smithi076 ceph-mon[24587]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:47 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:23:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:47 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:23:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:47 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:23:49.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:49 smithi022 ceph-mon[23758]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:49.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:49 smithi022 ceph-mon[28440]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:49 smithi076 ceph-mon[24587]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:51 smithi022 ceph-mon[23758]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:51 smithi022 ceph-mon[28440]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:51 smithi076 ceph-mon[24587]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:52.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:23:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:23:51] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.43.0" 2024-09-13T23:23:53.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:53 smithi022 ceph-mon[23758]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:53.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:53 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:53.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:53 smithi022 ceph-mon[28440]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:53.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:53 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:53.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:53 smithi076 ceph-mon[24587]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:53.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:53 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:23:55.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:55 smithi022 ceph-mon[23758]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:55 smithi022 ceph-mon[28440]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:55.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:55 smithi076 ceph-mon[24587]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:57.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:57 smithi022 ceph-mon[23758]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:57.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:57 smithi022 ceph-mon[28440]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:57 smithi076 ceph-mon[24587]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:23:59.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:23:59 smithi022 ceph-mon[23758]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:59.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:23:59 smithi022 ceph-mon[28440]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:23:59.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:23:59 smithi076 ceph-mon[24587]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:01.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:01 smithi022 ceph-mon[23758]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:01.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:01 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:24:01.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:01 smithi022 ceph-mon[28440]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:01.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:01 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:24:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:01 smithi076 ceph-mon[24587]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:01 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:24:02.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:24:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:24:01] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.43.0" 2024-09-13T23:24:03.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:03 smithi022 ceph-mon[23758]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:03.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:03 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:03.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:03 smithi022 ceph-mon[28440]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:03.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:03 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:03.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:03 smithi076 ceph-mon[24587]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:03.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:03 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:05.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:05 smithi022 ceph-mon[23758]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:05.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:05 smithi022 ceph-mon[28440]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:05 smithi076 ceph-mon[24587]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:07.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:07 smithi022 ceph-mon[23758]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:07 smithi022 ceph-mon[28440]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:07 smithi076 ceph-mon[24587]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:09.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:09 smithi022 ceph-mon[23758]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:09.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:09 smithi022 ceph-mon[28440]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:09 smithi076 ceph-mon[24587]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:11.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:11 smithi022 ceph-mon[23758]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:11.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:11 smithi022 ceph-mon[28440]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:11 smithi076 ceph-mon[24587]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:12.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:24:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:24:11] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.43.0" 2024-09-13T23:24:13.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:13 smithi022 ceph-mon[23758]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:13.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:13 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:13.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:13 smithi022 ceph-mon[28440]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:13.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:13 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:13.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:13 smithi076 ceph-mon[24587]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:13.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:13 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:15.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:15 smithi022 ceph-mon[23758]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:15 smithi022 ceph-mon[28440]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:15.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:15 smithi076 ceph-mon[24587]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:16.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:24:16.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:24:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:24:17.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:17 smithi022 ceph-mon[23758]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:17 smithi022 ceph-mon[28440]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:17.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:17 smithi076 ceph-mon[24587]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:19 smithi022 ceph-mon[23758]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:19 smithi022 ceph-mon[28440]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:19 smithi076 ceph-mon[24587]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:21.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:21 smithi022 ceph-mon[23758]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:21.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:21 smithi022 ceph-mon[28440]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:21 smithi076 ceph-mon[24587]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:22.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:24:21 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:24:21] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.43.0" 2024-09-13T23:24:23.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:23 smithi022 ceph-mon[23758]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:23.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:23 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:23.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:23 smithi022 ceph-mon[28440]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:23.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:23 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:23 smithi076 ceph-mon[24587]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:23 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:25.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:25 smithi022 ceph-mon[23758]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:25.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:25 smithi022 ceph-mon[28440]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:25 smithi076 ceph-mon[24587]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:27.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:27 smithi022 ceph-mon[23758]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:27.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:27 smithi022 ceph-mon[28440]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:27 smithi076 ceph-mon[24587]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:29.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:29 smithi022 ceph-mon[23758]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:29.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:29 smithi022 ceph-mon[28440]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:29 smithi076 ceph-mon[24587]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:31.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:31 smithi022 ceph-mon[28440]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:31 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:24:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:31 smithi022 ceph-mon[23758]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:31.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:31 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:24:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:31 smithi076 ceph-mon[24587]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:31 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:24:32.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:24:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:24:31] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-13T23:24:33.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:33 smithi022 ceph-mon[28440]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:33.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:33 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:33.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:33 smithi022 ceph-mon[23758]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:33.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:33 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:33.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:33 smithi076 ceph-mon[24587]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:33.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:33 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:35.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:35 smithi022 ceph-mon[23758]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:35.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:35 smithi022 ceph-mon[28440]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:35.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:35 smithi076 ceph-mon[24587]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:37 smithi076 ceph-mon[24587]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:37.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:37 smithi022 ceph-mon[23758]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:37 smithi022 ceph-mon[28440]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:39.479 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:39 smithi022 ceph-mon[23758]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:39.479 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:39 smithi022 ceph-mon[28440]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:39 smithi076 ceph-mon[24587]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:41 smithi076 ceph-mon[24587]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:41.606 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:41 smithi022 ceph-mon[23758]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:41.607 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:41 smithi022 ceph-mon[28440]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:42.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:24:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:24:41] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-13T23:24:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:43 smithi076 ceph-mon[24587]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:43 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:43.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:43 smithi022 ceph-mon[23758]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:43.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:43 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:43.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:43 smithi022 ceph-mon[28440]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:43.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:43 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:45 smithi076 ceph-mon[24587]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:45.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:45 smithi022 ceph-mon[23758]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:45.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:45 smithi022 ceph-mon[28440]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:46 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:24:46.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:46 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:24:46.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:46 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:24:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:47 smithi076 ceph-mon[24587]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:47.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:47 smithi022 ceph-mon[23758]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:47.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:47 smithi022 ceph-mon[28440]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:48 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:24:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:48 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:24:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:48 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:24:48.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:48 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:24:48.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:48 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:24:48.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:48 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:24:48.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:48 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:24:48.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:48 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:24:48.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:48 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:24:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:49 smithi076 ceph-mon[24587]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:49.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:49 smithi022 ceph-mon[23758]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:49.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:49 smithi022 ceph-mon[28440]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:51 smithi076 ceph-mon[24587]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:51.610 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:51 smithi022 ceph-mon[28440]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:51.611 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:51 smithi022 ceph-mon[23758]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:52.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:24:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:24:51] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.43.0" 2024-09-13T23:24:53.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:53 smithi076 ceph-mon[24587]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:53.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:53 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:53.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:53 smithi022 ceph-mon[23758]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:53.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:53 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:53.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:53 smithi022 ceph-mon[28440]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:53.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:53 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:24:55.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:55 smithi076 ceph-mon[24587]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:55.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:55 smithi022 ceph-mon[23758]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:55.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:55 smithi022 ceph-mon[28440]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:57 smithi076 ceph-mon[24587]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:57.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:57 smithi022 ceph-mon[23758]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:57.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:57 smithi022 ceph-mon[28440]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:24:59.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:24:59 smithi076 ceph-mon[24587]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:59.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:24:59 smithi022 ceph-mon[23758]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:24:59.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:24:59 smithi022 ceph-mon[28440]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:01 smithi076 ceph-mon[24587]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:01.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:01 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:01.616 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:01 smithi022 ceph-mon[28440]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:01.616 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:01 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:01.617 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:01 smithi022 ceph-mon[23758]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:01.617 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:01 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:02.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:25:01] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-13T23:25:03.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:03 smithi076 ceph-mon[24587]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:03.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:03 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:03.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:03 smithi022 ceph-mon[23758]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:03.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:03 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:03.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:03 smithi022 ceph-mon[28440]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:03.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:03 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:05 smithi076 ceph-mon[24587]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:05.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:05 smithi022 ceph-mon[23758]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:05.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:05 smithi022 ceph-mon[28440]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:07 smithi076 ceph-mon[24587]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:07.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:07 smithi022 ceph-mon[23758]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:07.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:07 smithi022 ceph-mon[28440]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:09 smithi076 ceph-mon[24587]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:09.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:09 smithi022 ceph-mon[23758]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:09.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:09 smithi022 ceph-mon[28440]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:11 smithi076 ceph-mon[24587]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:11.621 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:11 smithi022 ceph-mon[28440]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:11.622 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:11 smithi022 ceph-mon[23758]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:12.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:11 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:25:11] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-13T23:25:13.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:13 smithi076 ceph-mon[24587]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:13.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:13 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:13.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:13 smithi022 ceph-mon[23758]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:13.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:13 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:13.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:13 smithi022 ceph-mon[28440]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:13.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:13 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:15.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:15 smithi076 ceph-mon[24587]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:15.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:15 smithi022 ceph-mon[23758]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:15.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:15 smithi022 ceph-mon[28440]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:16 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:16.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:16 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:16.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:16 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:17.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:17 smithi076 ceph-mon[24587]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:17.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:17 smithi022 ceph-mon[23758]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:17.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:17 smithi022 ceph-mon[28440]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:18.642 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:25:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:19 smithi076 ceph-mon[24587]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:19.598 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:19 smithi022 ceph-mon[23758]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:19.599 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:19 smithi022 ceph-mon[28440]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:20.763 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:25:20.763 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (12m) 2m ago 20m 24.6M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:25:20.763 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (12m) 2m ago 20m 53.3M - 8.3.5 dad864ee21e9 969941fc6e23 2024-09-13T23:25:20.764 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (2m) 2m ago 19m 72.6M - 3.5 e1d6a67b021e 22d1d932c548 2024-09-13T23:25:20.764 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283 running (15m) 2m ago 25m 490M - 19.1.1-324-gecab7be5 0652ff2b5d78 194c1a9a8cc9 2024-09-13T23:25:20.764 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (9m) 2m ago 26m 437M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:25:20.764 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (26m) 2m ago 26m 69.7M 2048M 17.2.0 e1d6a67b021e 7d59bd9c71e8 2024-09-13T23:25:20.764 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (25m) 2m ago 25m 55.8M 2048M 17.2.0 e1d6a67b021e a1194b4b1c43 2024-09-13T23:25:20.764 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (25m) 2m ago 25m 50.2M 2048M 17.2.0 e1d6a67b021e 336a6f00a2a7 2024-09-13T23:25:20.764 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (12m) 2m ago 21m 15.9M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:25:20.764 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (12m) 2m ago 21m 16.3M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:25:20.764 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (24m) 2m ago 24m 51.5M 2993M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:25:20.764 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (24m) 2m ago 24m 54.0M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:25:20.765 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (24m) 2m ago 23m 50.2M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:25:20.765 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (23m) 2m ago 23m 51.4M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:25:20.765 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (23m) 2m ago 23m 53.2M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:25:20.765 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (22m) 2m ago 22m 54.9M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:25:20.765 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (22m) 2m ago 22m 54.1M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:25:20.765 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (22m) 2m ago 22m 53.4M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:25:20.765 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (2m) 2m ago 21m 42.2M - 2.43.0 a07b618ecd1d 959260f15170 2024-09-13T23:25:20.766 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (19m) 2m ago 19m 75.6M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:25:20.766 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (19m) 2m ago 19m 74.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:25:21.531 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:21 smithi022 ceph-mon[23758]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:21.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:21 smithi022 ceph-mon[28440]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:21 smithi076 ceph-mon[24587]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:21.560 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ls' 2024-09-13T23:25:22.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:21 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:25:21] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-09-13T23:25:22.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:22 smithi022 ceph-mon[23758]: from='client.25366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:22 smithi022 ceph-mon[28440]: from='client.25366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:22 smithi076 ceph-mon[24587]: from='client.25366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:23 smithi076 ceph-mon[24587]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:23 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:23.589 INFO:teuthology.orchestra.run.smithi022.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-13T23:25:23.589 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager ?:9093,9094 1/1 2m ago 21m smithi022=a;count:1 2024-09-13T23:25:23.589 INFO:teuthology.orchestra.run.smithi022.stdout:grafana ?:3000 1/1 2m ago 21m smithi076=a;count:1 2024-09-13T23:25:23.589 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo ?:5000 1/1 2m ago 19m count:1 2024-09-13T23:25:23.589 INFO:teuthology.orchestra.run.smithi022.stdout:mgr 2/2 2m ago 25m smithi022=y;smithi076=x;count:2 2024-09-13T23:25:23.589 INFO:teuthology.orchestra.run.smithi022.stdout:mon 3/3 2m ago 25m smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi076:172.21.15.76=b;count:3 2024-09-13T23:25:23.590 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter ?:9100 2/2 2m ago 21m smithi022=a;smithi076=b;count:2 2024-09-13T23:25:23.590 INFO:teuthology.orchestra.run.smithi022.stdout:osd 8 2m ago - 2024-09-13T23:25:23.590 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus ?:9095 1/1 2m ago 21m smithi076=a;count:1 2024-09-13T23:25:23.590 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo ?:8000 2/2 2m ago 19m count:2 2024-09-13T23:25:23.590 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:23 smithi022 ceph-mon[23758]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:23.590 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:23 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:23.590 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:23 smithi022 ceph-mon[28440]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:23.590 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:23 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:24.172 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T23:25:25.535 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:25 smithi022 ceph-mon[23758]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:25.536 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:25 smithi022 ceph-mon[23758]: from='client.15471 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:25.536 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:25 smithi022 ceph-mon[28440]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:25.536 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:25 smithi022 ceph-mon[28440]: from='client.15471 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:25 smithi076 ceph-mon[24587]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:25 smithi076 ceph-mon[24587]: from='client.15471 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:26.236 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:25:26.237 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:25:26.237 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T23:25:26.237 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:25:26.237 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:25:26.237 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:25:26.237 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:25:26.237 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:25:26.237 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T23:25:26.237 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:25:26.237 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": {}, 2024-09-13T23:25:26.237 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:25:26.237 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:25:26.238 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:25:26.238 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:25:26.238 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T23:25:26.238 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:25:26.238 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:25:26.238 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:25:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:26 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/479519856' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:25:26.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:26 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/479519856' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:25:26.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:26 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/479519856' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:25:26.884 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-09-13T23:25:27.471 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:27 smithi022 ceph-mon[23758]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:27.471 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:27 smithi022 ceph-mon[28440]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:27 smithi076 ceph-mon[24587]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:29 smithi076 ceph-mon[24587]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:29.578 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:29 smithi022 ceph-mon[23758]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:29.579 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:29 smithi022 ceph-mon[28440]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:25:30.260 INFO:teuthology.orchestra.run.smithi022.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:25:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:30 smithi076 ceph-mon[24587]: from='client.15483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:30 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:25:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:30 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:25:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:30 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:25:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:30 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:25:30.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:30 smithi022 ceph-mon[28440]: from='client.15483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:30.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:30 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:25:30.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:30 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:25:30.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:30 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:25:30.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:30 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:25:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:30 smithi022 ceph-mon[23758]: from='client.15483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:30 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:25:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:30 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:25:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:30 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:25:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:30 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:25:30.872 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T23:25:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:31 smithi076 ceph-mon[24587]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:31 smithi076 ceph-mon[24587]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:25:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:31 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:31.644 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:31 smithi022 ceph-mon[28440]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:31.644 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:31 smithi022 ceph-mon[28440]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:25:31.644 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:31 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:31.644 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:31 smithi022 ceph-mon[23758]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:31.644 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:31 smithi022 ceph-mon[23758]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:25:31.644 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:31 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:32.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:25:31] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-09-13T23:25:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:32 smithi076 ceph-mon[24587]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:25:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:32 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:25:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:32 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:25:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:32 smithi076 ceph-mon[24587]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T23:25:32.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:32 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T23:25:32.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:32 smithi076 ceph-mon[24587]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T23:25:32.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:32 smithi022 ceph-mon[23758]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:25:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:32 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:25:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:32 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:25:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:32 smithi022 ceph-mon[23758]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T23:25:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:32 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T23:25:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:32 smithi022 ceph-mon[23758]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T23:25:32.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:32 smithi022 ceph-mon[28440]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:25:32.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:32 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' 2024-09-13T23:25:32.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:32 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:25:32.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:32 smithi022 ceph-mon[28440]: from='mgr.35021 172.21.15.76:0/2700122295' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T23:25:32.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:32 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T23:25:32.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:32 smithi022 ceph-mon[28440]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T23:25:33.176 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:25:33] ENGINE Bus STOPPING 2024-09-13T23:25:33.421 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:33.063+0000 7f50c8719640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T23:25:33.421 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ignoring --setuser ceph since I am not root 2024-09-13T23:25:33.421 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ignoring --setgroup ceph since I am not root 2024-09-13T23:25:33.421 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mgr[44261]: -- 172.21.15.76:0/1636447601 <== mon.2 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55652d49b0e0 con 0x55652d478800 2024-09-13T23:25:33.421 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:33.215+0000 7ff200efa040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T23:25:33.421 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:33.271+0000 7ff200efa040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T23:25:33.421 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:33.418+0000 7ff200efa040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T23:25:33.422 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:33.422 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:33.422 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:25:33.422 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:25:33.422 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T23:25:33.422 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T23:25:33.422 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: Failing over to other MGR 2024-09-13T23:25:33.422 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: Standby manager daemon x started 2024-09-13T23:25:33.422 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.35021 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T23:25:33.422 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: mgrmap e36: y(active, starting, since 1.0057s), standbys: x 2024-09-13T23:25:33.422 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:25:33.422 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:25:33.423 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:25:33.423 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:25:33.423 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:25:33.423 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:25:33.423 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:25:33.423 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:25:33.423 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:25:33.423 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:25:33.423 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:25:33.423 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:25:33.423 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:25:33.424 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:25:33.424 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:25:33.424 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:25:33.521 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:33.521 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:33.521 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:25:33.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:25:33.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T23:25:33.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T23:25:33.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: Failing over to other MGR 2024-09-13T23:25:33.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: Standby manager daemon x started 2024-09-13T23:25:33.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.35021 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T23:25:33.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: mgrmap e36: y(active, starting, since 1.0057s), standbys: x 2024-09-13T23:25:33.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:25:33.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:25:33.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:25:33.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:25:33.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:25:33.523 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:25:33.523 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:25:33.523 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:25:33.523 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:25:33.523 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:25:33.523 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:25:33.523 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:25:33.523 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:25:33.523 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:25:33.523 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:25:33.523 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:25:33.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:25:33.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:33.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:25:33.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:25:33.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T23:25:33.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T23:25:33.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: Failing over to other MGR 2024-09-13T23:25:33.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: Standby manager daemon x started 2024-09-13T23:25:33.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.35021 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T23:25:33.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: mgrmap e36: y(active, starting, since 1.0057s), standbys: x 2024-09-13T23:25:33.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:25:33.524 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:25:33.525 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:25:33.526 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:25:33.777 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:33.484+0000 7ff200efa040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T23:25:33.777 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:33.634+0000 7ff200efa040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:25:33.777 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:33.775+0000 7ff200efa040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T23:25:33.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:25:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T23:25:33.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:25:33] ENGINE Bus STOPPED 2024-09-13T23:25:33.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:25:33] ENGINE Bus STARTING 2024-09-13T23:25:33.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:25:33] ENGINE Serving on http://:::9283 2024-09-13T23:25:33.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:25:33] ENGINE Bus STARTED 2024-09-13T23:25:34.325 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:25:34.371 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:34.098+0000 7ff200efa040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T23:25:34.371 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:34.163+0000 7ff200efa040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T23:25:34.371 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:34.369+0000 7ff200efa040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T23:25:34.371 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:34 smithi076 ceph-mon[24587]: Manager daemon y is now available 2024-09-13T23:25:34.371 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:34 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:34.372 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:34 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:25:34.372 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:34 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:25:34.372 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:34 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:25:34.372 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:34 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:25:34.662 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:34 smithi022 ceph-mon[23758]: Manager daemon y is now available 2024-09-13T23:25:34.662 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:34 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:34.662 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:34 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:25:34.662 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:34 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:25:34.662 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:34 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:25:34.662 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:34 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:25:34.663 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:34 smithi022 ceph-mon[28440]: Manager daemon y is now available 2024-09-13T23:25:34.663 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:34 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:34.663 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:34 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:25:34.663 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:34 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:25:34.663 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:34 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:25:34.663 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:34 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:25:35.056 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:25:35.056 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (12m) 2m ago 20m 24.6M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (12m) 2m ago 20m 53.3M - 8.3.5 dad864ee21e9 969941fc6e23 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (2m) 2m ago 19m 72.6M - 3.5 e1d6a67b021e 22d1d932c548 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283 running (15m) 2m ago 25m 490M - 19.1.1-324-gecab7be5 0652ff2b5d78 194c1a9a8cc9 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (9m) 2m ago 27m 437M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (27m) 2m ago 27m 69.7M 2048M 17.2.0 e1d6a67b021e 7d59bd9c71e8 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (25m) 2m ago 25m 55.8M 2048M 17.2.0 e1d6a67b021e a1194b4b1c43 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (25m) 2m ago 25m 50.2M 2048M 17.2.0 e1d6a67b021e 336a6f00a2a7 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (12m) 2m ago 21m 15.9M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (12m) 2m ago 21m 16.3M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (25m) 2m ago 25m 51.5M 2993M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (24m) 2m ago 24m 54.0M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (24m) 2m ago 24m 50.2M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:25:35.057 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (23m) 2m ago 23m 51.4M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:25:35.058 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (23m) 2m ago 23m 53.2M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:25:35.058 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (23m) 2m ago 23m 54.9M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:25:35.058 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (22m) 2m ago 22m 54.1M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:25:35.058 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (22m) 2m ago 22m 53.4M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:25:35.058 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (3m) 2m ago 21m 42.2M - 2.43.0 a07b618ecd1d 959260f15170 2024-09-13T23:25:35.058 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (20m) 2m ago 20m 75.6M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:25:35.058 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (19m) 2m ago 19m 74.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:25:35.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-mon[23758]: mgrmap e37: y(active, since 2s), standbys: x 2024-09-13T23:25:35.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-mon[23758]: from='client.35201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:35.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-mon[23758]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:25:35.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-mon[28440]: mgrmap e37: y(active, since 2s), standbys: x 2024-09-13T23:25:35.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-mon[28440]: from='client.35201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:35.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-mon[28440]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:25:35.536 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:25:35.536 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:25:35.537 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T23:25:35.537 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:25:35.537 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:25:35.537 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:25:35.537 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:25:35.537 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:25:35.537 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T23:25:35.537 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:25:35.537 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": {}, 2024-09-13T23:25:35.537 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:25:35.537 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:25:35.538 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:25:35.538 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:25:35.538 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T23:25:35.538 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:25:35.538 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:25:35.538 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:25:35.603 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:35 smithi076 ceph-mon[24587]: mgrmap e37: y(active, since 2s), standbys: x 2024-09-13T23:25:35.603 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:35 smithi076 ceph-mon[24587]: from='client.35201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:35.603 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:35 smithi076 ceph-mon[24587]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:25:35.604 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:35.352+0000 7ff200efa040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T23:25:35.604 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:35.423+0000 7ff200efa040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T23:25:35.898 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:25:35.898 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:25:35.898 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:25:35.898 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T23:25:35.898 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-13T23:25:35.898 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-13T23:25:35.898 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-13T23:25:35.898 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "2/2 daemons upgraded", 2024-09-13T23:25:35.898 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-09-13T23:25:35.899 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:25:35.899 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:25:35.899 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:25:35.669+0000 7f7e2c036640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:23:25:35] ENGINE Error in HTTPServer.serve 2024-09-13T23:25:35.899 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: Traceback (most recent call last): 2024-09-13T23:25:35.899 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:25:35.899 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: self._connections.run(self.expiration_interval) 2024-09-13T23:25:35.899 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:25:35.899 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: self._run(expiration_interval) 2024-09-13T23:25:35.900 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:25:35.900 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:25:35.900 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:25:35.900 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:25:35.900 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:25:35.900 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: s = self.context.wrap_socket( 2024-09-13T23:25:35.900 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:25:35.900 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: return self.sslsocket_class._create( 2024-09-13T23:25:35.900 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:25:35.900 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: self.do_handshake() 2024-09-13T23:25:35.901 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:25:35.901 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: self._sslobj.do_handshake() 2024-09-13T23:25:35.901 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:25:35.901 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:25:35 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:25:36.465 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:36.194+0000 7ff200efa040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T23:25:36.465 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:36.318+0000 7ff200efa040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T23:25:36.465 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-mon[24587]: from='client.35207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:36.465 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-mon[24587]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:36.465 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-mon[24587]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:25:36.465 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-mon[24587]: mgrmap e38: y(active, since 3s), standbys: x 2024-09-13T23:25:36.465 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/35279787' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:25:36.625 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:36 smithi022 ceph-mon[28440]: from='client.35207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:36.626 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:36 smithi022 ceph-mon[28440]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:36.626 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:36 smithi022 ceph-mon[28440]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:25:36.626 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:36 smithi022 ceph-mon[28440]: mgrmap e38: y(active, since 3s), standbys: x 2024-09-13T23:25:36.626 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:36 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/35279787' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:25:36.626 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:36 smithi022 ceph-mon[23758]: from='client.35207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:36.626 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:36 smithi022 ceph-mon[23758]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:36.626 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:36 smithi022 ceph-mon[23758]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:25:36.627 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:36 smithi022 ceph-mon[23758]: mgrmap e38: y(active, since 3s), standbys: x 2024-09-13T23:25:36.627 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:36 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/35279787' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:25:36.749 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:36.463+0000 7ff200efa040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:25:36.749 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:36.526+0000 7ff200efa040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T23:25:36.749 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:36.675+0000 7ff200efa040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T23:25:36.749 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:36.747+0000 7ff200efa040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T23:25:37.010 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:36.817+0000 7ff200efa040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T23:25:37.010 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:36.979+0000 7ff200efa040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T23:25:37.264 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:37.048+0000 7ff200efa040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T23:25:37.264 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:37.117+0000 7ff200efa040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T23:25:37.264 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:37.201+0000 7ff200efa040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T23:25:37.264 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:37.263+0000 7ff200efa040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T23:25:37.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:37 smithi022 ceph-mon[23758]: [13/Sep/2024:23:25:35] ENGINE Bus STARTING 2024-09-13T23:25:37.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:37 smithi022 ceph-mon[23758]: [13/Sep/2024:23:25:35] ENGINE Serving on http://172.21.15.22:8765 2024-09-13T23:25:37.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:37 smithi022 ceph-mon[23758]: [13/Sep/2024:23:25:35] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:25:37.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:37 smithi022 ceph-mon[23758]: [13/Sep/2024:23:25:35] ENGINE Bus STARTED 2024-09-13T23:25:37.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:37 smithi022 ceph-mon[23758]: [13/Sep/2024:23:25:35] ENGINE Error in HTTPServer.serve 2024-09-13T23:25:37.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:25:37.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:25:37.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:25:37.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:25:37.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._run(expiration_interval) 2024-09-13T23:25:37.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:25:37.427 INFO:journalctl@ceph.mon.a.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:25:37.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:25:37.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:25:37.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:25:37.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: s = self.context.wrap_socket( 2024-09-13T23:25:37.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:25:37.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.sslsocket_class._create( 2024-09-13T23:25:37.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:25:37.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.do_handshake() 2024-09-13T23:25:37.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:25:37.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._sslobj.do_handshake() 2024-09-13T23:25:37.428 INFO:journalctl@ceph.mon.a.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:25:37.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:37 smithi022 ceph-mon[23758]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:37.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:37 smithi022 ceph-mon[28440]: [13/Sep/2024:23:25:35] ENGINE Bus STARTING 2024-09-13T23:25:37.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:37 smithi022 ceph-mon[28440]: [13/Sep/2024:23:25:35] ENGINE Serving on http://172.21.15.22:8765 2024-09-13T23:25:37.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:37 smithi022 ceph-mon[28440]: [13/Sep/2024:23:25:35] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:25:37.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:37 smithi022 ceph-mon[28440]: [13/Sep/2024:23:25:35] ENGINE Bus STARTED 2024-09-13T23:25:37.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:37 smithi022 ceph-mon[28440]: [13/Sep/2024:23:25:35] ENGINE Error in HTTPServer.serve 2024-09-13T23:25:37.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:25:37.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:25:37.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:25:37.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:25:37.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._run(expiration_interval) 2024-09-13T23:25:37.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:25:37.429 INFO:journalctl@ceph.mon.c.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:25:37.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:25:37.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:25:37.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:25:37.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: s = self.context.wrap_socket( 2024-09-13T23:25:37.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:25:37.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.sslsocket_class._create( 2024-09-13T23:25:37.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:25:37.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.do_handshake() 2024-09-13T23:25:37.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:25:37.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._sslobj.do_handshake() 2024-09-13T23:25:37.430 INFO:journalctl@ceph.mon.c.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:25:37.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:37 smithi022 ceph-mon[28440]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:37.540 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:37.328+0000 7ff200efa040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T23:25:37.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:37 smithi076 ceph-mon[24587]: [13/Sep/2024:23:25:35] ENGINE Bus STARTING 2024-09-13T23:25:37.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:37 smithi076 ceph-mon[24587]: [13/Sep/2024:23:25:35] ENGINE Serving on http://172.21.15.22:8765 2024-09-13T23:25:37.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:37 smithi076 ceph-mon[24587]: [13/Sep/2024:23:25:35] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:25:37.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:37 smithi076 ceph-mon[24587]: [13/Sep/2024:23:25:35] ENGINE Bus STARTED 2024-09-13T23:25:37.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:37 smithi076 ceph-mon[24587]: [13/Sep/2024:23:25:35] ENGINE Error in HTTPServer.serve 2024-09-13T23:25:37.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:25:37.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:25:37.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:25:37.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:25:37.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._run(expiration_interval) 2024-09-13T23:25:37.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:25:37.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:25:37.541 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:25:37.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:25:37.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:25:37.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: s = self.context.wrap_socket( 2024-09-13T23:25:37.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:25:37.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.sslsocket_class._create( 2024-09-13T23:25:37.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:25:37.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: self.do_handshake() 2024-09-13T23:25:37.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:25:37.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._sslobj.do_handshake() 2024-09-13T23:25:37.542 INFO:journalctl@ceph.mon.b.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:25:37.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:37 smithi076 ceph-mon[24587]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:25:38.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:38.157+0000 7ff200efa040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T23:25:38.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:25:38] ENGINE Bus STARTING 2024-09-13T23:25:38.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: CherryPy Checker: 2024-09-13T23:25:38.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: The Application mounted at '' has an empty config. 2024-09-13T23:25:38.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: 2024-09-13T23:25:38.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:25:38] ENGINE Serving on http://:::9283 2024-09-13T23:25:38.288 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: [13/Sep/2024:23:25:38] ENGINE Bus STARTED 2024-09-13T23:25:38.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[23758]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:25:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[23758]: mgrmap e39: y(active, since 5s), standbys: x 2024-09-13T23:25:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[23758]: Standby manager daemon x restarted 2024-09-13T23:25:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[23758]: Standby manager daemon x started 2024-09-13T23:25:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2634681966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:25:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2634681966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:25:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2634681966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:25:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/2634681966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:25:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:38.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[28440]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:25:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[28440]: mgrmap e39: y(active, since 5s), standbys: x 2024-09-13T23:25:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[28440]: Standby manager daemon x restarted 2024-09-13T23:25:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[28440]: Standby manager daemon x started 2024-09-13T23:25:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2634681966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:25:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2634681966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:25:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2634681966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:25:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/2634681966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:25:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:38.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:38 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:38.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-mon[24587]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:25:38.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-mon[24587]: mgrmap e39: y(active, since 5s), standbys: x 2024-09-13T23:25:38.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-mon[24587]: Standby manager daemon x restarted 2024-09-13T23:25:38.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-mon[24587]: Standby manager daemon x started 2024-09-13T23:25:38.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2634681966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:25:38.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2634681966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:25:38.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2634681966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:25:38.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/2634681966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:25:38.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:38.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:38 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[23758]: mgrmap e40: y(active, since 7s), standbys: x 2024-09-13T23:25:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[23758]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:25:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:25:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:25:40.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[28440]: mgrmap e40: y(active, since 7s), standbys: x 2024-09-13T23:25:40.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[28440]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:25:40.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:25:40.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:39 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:25:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:39 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:39 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:39 smithi076 ceph-mon[24587]: mgrmap e40: y(active, since 7s), standbys: x 2024-09-13T23:25:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:39 smithi076 ceph-mon[24587]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:25:40.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:39 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:39 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:39 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:39 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:40.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:39 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:25:40.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:39 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:25:41.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:25:41.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:25:41.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:25:41.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:25:41.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:25:41.423 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:25:41.423 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.423 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.423 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.423 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.423 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.423 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:25:41.423 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:25:41.423 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:25:41.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:25:41.424 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:25:41.424 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:25:41.424 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:25:41.424 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:25:41.424 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:25:41.424 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.424 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.424 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.424 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.424 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.424 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:25:41.424 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:25:41.425 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:41 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:25:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:25:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:25:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:25:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:25:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:25:41.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:25:41.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:41.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:25:41.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:25:41.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:25:42.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[44257]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:25:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T23:25:42.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:42 smithi022 ceph-mon[23758]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:25:42.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:42 smithi022 ceph-mon[23758]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:25:42.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:42 smithi022 ceph-mon[23758]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:25:42.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:42 smithi022 ceph-mon[23758]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:25:42.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:42 smithi022 ceph-mon[23758]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 158 KiB/s rd, 0 B/s wr, 229 op/s 2024-09-13T23:25:42.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:42 smithi022 ceph-mon[28440]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:25:42.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:42 smithi022 ceph-mon[28440]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:25:42.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:42 smithi022 ceph-mon[28440]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:25:42.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:42 smithi022 ceph-mon[28440]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:25:42.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:42 smithi022 ceph-mon[28440]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 158 KiB/s rd, 0 B/s wr, 229 op/s 2024-09-13T23:25:42.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:42 smithi076 ceph-mon[24587]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:25:42.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:42 smithi076 ceph-mon[24587]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:25:42.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:42 smithi076 ceph-mon[24587]: Reconfiguring iscsi.foo.smithi022.wguwxm (dependencies changed)... 2024-09-13T23:25:42.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:42 smithi076 ceph-mon[24587]: Reconfiguring daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:25:42.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:42 smithi076 ceph-mon[24587]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 158 KiB/s rd, 0 B/s wr, 229 op/s 2024-09-13T23:25:43.356 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:43 smithi022 ceph-mon[23758]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:43.356 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:43 smithi022 ceph-mon[28440]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:43.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:43 smithi076 ceph-mon[24587]: from='client.35141 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:44.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:44 smithi022 ceph-mon[23758]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 122 KiB/s rd, 0 B/s wr, 176 op/s 2024-09-13T23:25:44.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:44 smithi022 ceph-mon[28440]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 122 KiB/s rd, 0 B/s wr, 176 op/s 2024-09-13T23:25:44.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:44 smithi076 ceph-mon[24587]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 122 KiB/s rd, 0 B/s wr, 176 op/s 2024-09-13T23:25:46.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:46 smithi022 ceph-mon[23758]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 0 B/s wr, 144 op/s 2024-09-13T23:25:46.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:46 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:46.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:46 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:46.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:46 smithi022 ceph-mon[23758]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:25:46.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:46 smithi022 ceph-mon[23758]: Reconfiguring daemon prometheus.a on smithi076 2024-09-13T23:25:46.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:46 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3022971651' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:25:46.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:46 smithi022 ceph-mon[28440]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 0 B/s wr, 144 op/s 2024-09-13T23:25:46.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:46 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:46.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:46 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:46.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:46 smithi022 ceph-mon[28440]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:25:46.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:46 smithi022 ceph-mon[28440]: Reconfiguring daemon prometheus.a on smithi076 2024-09-13T23:25:46.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:46 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3022971651' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:25:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:46 smithi076 ceph-mon[24587]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 0 B/s wr, 144 op/s 2024-09-13T23:25:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:46 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:46 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:46 smithi076 ceph-mon[24587]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T23:25:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:46 smithi076 ceph-mon[24587]: Reconfiguring daemon prometheus.a on smithi076 2024-09-13T23:25:46.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:46 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3022971651' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:25:48.283 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-mon[24587]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-09-13T23:25:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:48 smithi022 ceph-mon[23758]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-09-13T23:25:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:48 smithi022 ceph-mon[28440]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-09-13T23:25:48.537 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 systemd[1]: Stopping Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:25:48.947 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:25:48.645Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T23:25:48.947 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:25:48.645Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T23:25:48.947 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:25:48.645Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T23:25:48.947 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:25:48.645Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T23:25:48.947 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:25:48.645Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T23:25:48.947 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:25:48.645Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T23:25:48.947 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:25:48.645Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T23:25:48.947 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:25:48.645Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T23:25:48.947 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:25:48.645Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T23:25:48.948 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:25:48.645Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T23:25:48.948 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:25:48.645Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T23:25:48.948 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[56665]: ts=2024-09-13T23:25:48.645Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T23:25:48.948 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:48 smithi076 podman[59445]: 2024-09-13 23:25:48.655454085 +0000 UTC m=+0.109708690 container died 959260f15170e55f4c342c455bd3eeeac670987d4cd2a6a01816c038b23490a4 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:25:49.262 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 podman[59445]: 2024-09-13 23:25:49.044796158 +0000 UTC m=+0.499050794 container cleanup 959260f15170e55f4c342c455bd3eeeac670987d4cd2a6a01816c038b23490a4 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:25:49.263 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 bash[59445]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T23:25:49.263 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:49.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:49 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:49.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:49 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:25:49.537 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 podman[59457]: 2024-09-13 23:25:49.262321362 +0000 UTC m=+0.608357518 container remove 959260f15170e55f4c342c455bd3eeeac670987d4cd2a6a01816c038b23490a4 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:25:49.537 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T23:25:49.537 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 systemd[1]: Stopped Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:25:49.537 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.724s CPU time. 2024-09-13T23:25:49.537 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 systemd[1]: Starting Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:25:49.932 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 podman[59551]: 2024-09-13 23:25:49.533935277 +0000 UTC m=+0.017249914 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T23:25:49.932 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 podman[59551]: 2024-09-13 23:25:49.663351914 +0000 UTC m=+0.146666554 container create 6f8a2fca1117fdc337860919191ee0142cb9f8dd52f08418ed756c86b58a4cfd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:25:49.932 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 podman[59551]: 2024-09-13 23:25:49.932073003 +0000 UTC m=+0.415387649 container init 6f8a2fca1117fdc337860919191ee0142cb9f8dd52f08418ed756c86b58a4cfd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:25:50.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:50 smithi022 ceph-mon[23758]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-09-13T23:25:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:50 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:50 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:50.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:50 smithi022 ceph-mon[28440]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-09-13T23:25:50.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:50 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:50.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:50 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:50.274 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:50 smithi076 ceph-mon[24587]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-09-13T23:25:50.275 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:50 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:50.275 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:50 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:50.275 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 podman[59551]: 2024-09-13 23:25:49.934943725 +0000 UTC m=+0.418258372 container start 6f8a2fca1117fdc337860919191ee0142cb9f8dd52f08418ed756c86b58a4cfd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:25:50.275 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.958Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T23:25:50.275 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.958Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T23:25:50.275 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.958Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi076 (none))" 2024-09-13T23:25:50.276 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.958Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T23:25:50.276 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.958Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T23:25:50.276 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.959Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T23:25:50.276 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.960Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T23:25:50.276 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.961Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T23:25:50.276 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.961Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T23:25:50.276 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.962Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T23:25:50.276 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.962Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.2µs 2024-09-13T23:25:50.276 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.962Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T23:25:50.276 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.969Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-09-13T23:25:50.276 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.982Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-09-13T23:25:50.276 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.992Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-09-13T23:25:50.277 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.994Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-09-13T23:25:50.277 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.996Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-09-13T23:25:50.277 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.996Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-09-13T23:25:50.277 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.996Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.825µs wal_replay_duration=33.984816ms wbl_replay_duration=103ns total_replay_duration=34.054543ms 2024-09-13T23:25:50.277 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.999Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T23:25:50.277 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.999Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T23:25:50.277 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:49.999Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T23:25:50.277 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:50.020Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.263957ms db_storage=813ns remote_storage=1.548µs web_handler=427ns query_engine=590ns scrape=382.653µs scrape_sd=102.763µs notify=9.791µs notify_sd=8.377µs rules=19.442219ms tracing=7.883µs 2024-09-13T23:25:50.277 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:50.020Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T23:25:50.277 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:25:50.020Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T23:25:50.277 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:50 smithi076 bash[59551]: 6f8a2fca1117fdc337860919191ee0142cb9f8dd52f08418ed756c86b58a4cfd 2024-09-13T23:25:50.277 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:25:50 smithi076 systemd[1]: Started Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:25:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:25:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:25:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[23758]: Adding iSCSI gateway http://:@172.21.15.22:5000 to Dashboard 2024-09-13T23:25:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:25:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:25:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:25:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:25:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:51.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:25:51.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:25:51.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:25:51.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:25:51.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:25:51.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:51.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[28440]: Adding iSCSI gateway http://:@172.21.15.22:5000 to Dashboard 2024-09-13T23:25:51.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:25:51.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:25:51.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:25:51.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:25:51.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:51.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:25:51.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:25:51.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:51 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:25:51.439 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:51 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:25:51.439 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:51 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T23:25:51.439 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:51 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:51.439 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:51 smithi076 ceph-mon[24587]: Adding iSCSI gateway http://:@172.21.15.22:5000 to Dashboard 2024-09-13T23:25:51.439 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:51 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:25:51.439 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:51 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T23:25:51.439 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:51 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:25:51.440 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:51 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi022"}]: dispatch 2024-09-13T23:25:51.440 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:51 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:51.440 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:51 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:25:51.440 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:51 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T23:25:51.440 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:51 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:25:52.092 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:52 smithi076 ceph-mon[24587]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 0 B/s wr, 130 op/s 2024-09-13T23:25:52.092 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:52 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:25:52.092 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:52 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:25:52.092 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:52 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:25:52.092 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:52 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:25:52.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:52 smithi022 ceph-mon[28440]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 0 B/s wr, 130 op/s 2024-09-13T23:25:52.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:52 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:25:52.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:52 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:25:52.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:52 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:25:52.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:52 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:25:52.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:52 smithi022 ceph-mon[23758]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 0 B/s wr, 130 op/s 2024-09-13T23:25:52.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:52 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:25:52.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:52 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:25:52.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:52 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:25:52.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:52 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:25:53.162 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:53 smithi076 ceph-mon[24587]: Upgrade: Updating mgr.x 2024-09-13T23:25:53.162 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:53 smithi076 ceph-mon[24587]: Deploying daemon mgr.x on smithi076 2024-09-13T23:25:53.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:53 smithi022 ceph-mon[28440]: Upgrade: Updating mgr.x 2024-09-13T23:25:53.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:53 smithi022 ceph-mon[28440]: Deploying daemon mgr.x on smithi076 2024-09-13T23:25:53.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:53 smithi022 ceph-mon[23758]: Upgrade: Updating mgr.x 2024-09-13T23:25:53.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:53 smithi022 ceph-mon[23758]: Deploying daemon mgr.x on smithi076 2024-09-13T23:25:53.438 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:53 smithi076 systemd[1]: Stopping Ceph mgr.x for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:25:53.774 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:53 smithi076 podman[59899]: 2024-09-13 23:25:53.437898995 +0000 UTC m=+0.130611874 container died 194c1a9a8cc9fcc052fc89adbbae3261eeb35cee1dba6eca226117dfb6132bf3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T23:25:54.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:53 smithi076 podman[59899]: 2024-09-13 23:25:53.773126068 +0000 UTC m=+0.465838962 container cleanup 194c1a9a8cc9fcc052fc89adbbae3261eeb35cee1dba6eca226117dfb6132bf3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, ceph=True, org.label-schema.schema-version=1.0) 2024-09-13T23:25:54.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:53 smithi076 bash[59899]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x 2024-09-13T23:25:54.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:54 smithi076 podman[59911]: 2024-09-13 23:25:54.007753111 +0000 UTC m=+0.567281297 container remove 194c1a9a8cc9fcc052fc89adbbae3261eeb35cee1dba6eca226117dfb6132bf3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=) 2024-09-13T23:25:54.037 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:54 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T23:25:54.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:54 smithi022 ceph-mon[23758]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-09-13T23:25:54.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:54 smithi022 ceph-mon[28440]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-09-13T23:25:54.451 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:54 smithi076 ceph-mon[24587]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-09-13T23:25:54.451 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:54 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T23:25:54.451 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:54 smithi076 systemd[1]: Stopped Ceph mgr.x for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:25:54.451 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:54 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.x.service: Consumed 31.793s CPU time. 2024-09-13T23:25:54.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:54 smithi076 systemd[1]: Starting Ceph mgr.x for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:25:54.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:54 smithi076 podman[60043]: 2024-09-13 23:25:54.563270894 +0000 UTC m=+0.016170074 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:25:54.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:54 smithi076 podman[60043]: 2024-09-13 23:25:54.692516868 +0000 UTC m=+0.145416034 container create 8f0261a79cc0e18d3efedfd9cb0e0d326499a1c2ed3686ccc2fef7d44f19cf6c (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-13T23:25:55.072 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:54 smithi076 podman[60043]: 2024-09-13 23:25:54.944973937 +0000 UTC m=+0.397873100 container init 8f0261a79cc0e18d3efedfd9cb0e0d326499a1c2ed3686ccc2fef7d44f19cf6c (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:25:55.072 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:54 smithi076 podman[60043]: 2024-09-13 23:25:54.947994922 +0000 UTC m=+0.400894091 container start 8f0261a79cc0e18d3efedfd9cb0e0d326499a1c2ed3686ccc2fef7d44f19cf6c (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-13T23:25:55.072 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:54 smithi076 ceph-mgr[60062]: -- 172.21.15.76:0/1174188532 <== mon.2 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a104d990e0 con 0x55a104d76800 2024-09-13T23:25:55.072 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:55 smithi076 bash[60043]: 8f0261a79cc0e18d3efedfd9cb0e0d326499a1c2ed3686ccc2fef7d44f19cf6c 2024-09-13T23:25:55.072 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:55 smithi076 systemd[1]: Started Ceph mgr.x for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:25:55.368 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:55.069+0000 7efe7cf59040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T23:25:55.368 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:55.134+0000 7efe7cf59040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T23:25:55.369 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:55.295+0000 7efe7cf59040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T23:25:55.369 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:55.366+0000 7efe7cf59040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T23:25:55.673 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:55.531+0000 7efe7cf59040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:25:55.673 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:55.671+0000 7efe7cf59040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T23:25:56.265 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:55.997+0000 7efe7cf59040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T23:25:56.265 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:56.058+0000 7efe7cf59040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T23:25:56.265 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:56 smithi076 ceph-mon[24587]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T23:25:56.266 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:56 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:56.266 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:56 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:56.266 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:56 smithi076 ceph-mon[24587]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:56.377 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:56 smithi022 ceph-mon[23758]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T23:25:56.377 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:56 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:56.377 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:56 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:56.377 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:56 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:56.378 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:56 smithi022 ceph-mon[28440]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T23:25:56.378 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:56 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:56.378 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:56 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:25:56.378 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:56 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:25:56.537 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:56.263+0000 7efe7cf59040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T23:25:57.412 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:57.208+0000 7efe7cf59040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T23:25:57.412 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:57.279+0000 7efe7cf59040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T23:25:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:25:58 smithi076 ceph-mon[24587]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:25:58.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:58.020+0000 7efe7cf59040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T23:25:58.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:58.140+0000 7efe7cf59040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T23:25:58.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:25:58 smithi022 ceph-mon[23758]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:25:58.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:25:58 smithi022 ceph-mon[28440]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:25:58.583 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:58.286+0000 7efe7cf59040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:25:58.583 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:58.348+0000 7efe7cf59040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T23:25:58.583 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:58.501+0000 7efe7cf59040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T23:25:58.583 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:58.581+0000 7efe7cf59040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T23:25:58.875 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:58.641+0000 7efe7cf59040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T23:25:58.875 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:58.807+0000 7efe7cf59040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T23:25:58.875 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:58.873+0000 7efe7cf59040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T23:25:59.145 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:58.939+0000 7efe7cf59040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T23:25:59.145 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:59.022+0000 7efe7cf59040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T23:25:59.145 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:59.083+0000 7efe7cf59040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T23:25:59.145 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:59.143+0000 7efe7cf59040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T23:26:00.249 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:25:59.992+0000 7efe7cf59040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T23:26:00.249 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: [13/Sep/2024:23:25:59] ENGINE Bus STARTING 2024-09-13T23:26:00.249 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: CherryPy Checker: 2024-09-13T23:26:00.249 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: The Application mounted at '' has an empty config. 2024-09-13T23:26:00.249 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:25:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:26:00.249 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: [13/Sep/2024:23:26:00] ENGINE Serving on http://:::9283 2024-09-13T23:26:00.249 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: [13/Sep/2024:23:26:00] ENGINE Bus STARTED 2024-09-13T23:26:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:26:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: Standby manager daemon x restarted 2024-09-13T23:26:00.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: Standby manager daemon x started 2024-09-13T23:26:00.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/1038351001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:26:00.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/1038351001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:26:00.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/1038351001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:26:00.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:00 smithi076 ceph-mon[24587]: from='mgr.? 172.21.15.76:0/1038351001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:26:00.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:26:00.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: Standby manager daemon x restarted 2024-09-13T23:26:00.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: Standby manager daemon x started 2024-09-13T23:26:00.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/1038351001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:26:00.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/1038351001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:26:00.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/1038351001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:26:00.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[23758]: from='mgr.? 172.21.15.76:0/1038351001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:26:00.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:26:00.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:00.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: Standby manager daemon x restarted 2024-09-13T23:26:00.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: Standby manager daemon x started 2024-09-13T23:26:00.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/1038351001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:26:00.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/1038351001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:26:00.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/1038351001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:26:00.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:00 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/1038351001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:26:02.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:01 smithi022 ceph-mon[23758]: mgrmap e41: y(active, since 28s), standbys: x 2024-09-13T23:26:02.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:01 smithi022 ceph-mon[23758]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:02.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:01 smithi022 ceph-mon[28440]: mgrmap e41: y(active, since 28s), standbys: x 2024-09-13T23:26:02.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:01 smithi022 ceph-mon[28440]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:01 smithi076 ceph-mon[24587]: mgrmap e41: y(active, since 28s), standbys: x 2024-09-13T23:26:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:01 smithi076 ceph-mon[24587]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:02.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:26:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:26:02] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-13T23:26:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T23:26:04.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:26:04.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:04.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:26:04.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:26:04.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:04.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:26:04.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:26:04.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:26:04.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:04.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:04.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:26:05.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all mgr 2024-09-13T23:26:05.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all crash 2024-09-13T23:26:05.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all mds 2024-09-13T23:26:05.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:26:05.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T23:26:05.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:26:05.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all nfs 2024-09-13T23:26:05.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:26:05.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all node-exporter 2024-09-13T23:26:05.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all prometheus 2024-09-13T23:26:05.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all alertmanager 2024-09-13T23:26:05.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all mgr 2024-09-13T23:26:05.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all crash 2024-09-13T23:26:05.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all mds 2024-09-13T23:26:05.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:26:05.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T23:26:05.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:26:05.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all nfs 2024-09-13T23:26:05.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:26:05.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all node-exporter 2024-09-13T23:26:05.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all prometheus 2024-09-13T23:26:05.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:05 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all alertmanager 2024-09-13T23:26:05.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:05 smithi076 ceph-mon[24587]: Upgrade: Setting container_image for all mgr 2024-09-13T23:26:05.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:05 smithi076 ceph-mon[24587]: Upgrade: Setting container_image for all crash 2024-09-13T23:26:05.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:05 smithi076 ceph-mon[24587]: Upgrade: Setting container_image for all mds 2024-09-13T23:26:05.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:05 smithi076 ceph-mon[24587]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:26:05.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:05 smithi076 ceph-mon[24587]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T23:26:05.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:05 smithi076 ceph-mon[24587]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:26:05.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:05 smithi076 ceph-mon[24587]: Upgrade: Setting container_image for all nfs 2024-09-13T23:26:05.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:05 smithi076 ceph-mon[24587]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:26:05.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:05 smithi076 ceph-mon[24587]: Upgrade: Setting container_image for all node-exporter 2024-09-13T23:26:05.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:05 smithi076 ceph-mon[24587]: Upgrade: Setting container_image for all prometheus 2024-09-13T23:26:05.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:05 smithi076 ceph-mon[24587]: Upgrade: Setting container_image for all alertmanager 2024-09-13T23:26:06.280 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:26:06.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:06 smithi022 ceph-mon[23758]: Upgrade: Updating grafana.a 2024-09-13T23:26:06.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:06 smithi022 ceph-mon[23758]: Deploying daemon grafana.a on smithi076 2024-09-13T23:26:06.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:06 smithi022 ceph-mon[23758]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:06.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:06 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:06.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:06 smithi022 ceph-mon[28440]: Upgrade: Updating grafana.a 2024-09-13T23:26:06.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:06 smithi022 ceph-mon[28440]: Deploying daemon grafana.a on smithi076 2024-09-13T23:26:06.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:06 smithi022 ceph-mon[28440]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:06.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:06 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:06.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:06 smithi076 ceph-mon[24587]: Upgrade: Updating grafana.a 2024-09-13T23:26:06.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:06 smithi076 ceph-mon[24587]: Deploying daemon grafana.a on smithi076 2024-09-13T23:26:06.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:06 smithi076 ceph-mon[24587]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:06.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:06 smithi076 ceph-mon[24587]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:06.961 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:26:06.961 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (13m) 7s ago 21m 24.7M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:26:06.961 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (12m) 7s ago 21m 53.8M - 8.3.5 dad864ee21e9 969941fc6e23 2024-09-13T23:26:06.961 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (22s) 7s ago 20m 50.4M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:26:06.961 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (12s) 7s ago 25m 66.7M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:26:06.961 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (9m) 7s ago 27m 490M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:26:06.961 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (27m) 7s ago 27m 75.5M 2048M 17.2.0 e1d6a67b021e 7d59bd9c71e8 2024-09-13T23:26:06.961 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (26m) 7s ago 26m 60.1M 2048M 17.2.0 e1d6a67b021e a1194b4b1c43 2024-09-13T23:26:06.961 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (26m) 7s ago 26m 54.4M 2048M 17.2.0 e1d6a67b021e 336a6f00a2a7 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (13m) 7s ago 21m 16.0M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (12m) 7s ago 21m 16.5M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (25m) 7s ago 25m 52.5M 2993M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (25m) 7s ago 25m 55.4M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (24m) 7s ago 24m 50.7M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (24m) 7s ago 24m 53.1M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (23m) 7s ago 23m 55.7M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (23m) 7s ago 23m 55.8M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (23m) 7s ago 23m 54.7M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (22m) 7s ago 22m 54.2M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (17s) 7s ago 22m 47.9M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (20m) 7s ago 20m 76.4M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:26:06.962 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (20m) 7s ago 20m 75.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:26:07.377 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:26:07.377 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:26:07.377 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T23:26:07.377 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:26:07.377 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:26:07.377 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:26:07.377 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:26:07.377 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:26:07.377 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T23:26:07.377 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:26:07.378 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": {}, 2024-09-13T23:26:07.378 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:26:07.378 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:26:07.378 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:26:07.378 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:26:07.378 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T23:26:07.378 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:26:07.378 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:26:07.378 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:26:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:07 smithi076 ceph-mon[24587]: from='client.25474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:07.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:07 smithi022 ceph-mon[28440]: from='client.25474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:07.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:07 smithi022 ceph-mon[23758]: from='client.25474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:07.728 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:26:07.729 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:26:07.729 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:26:07.729 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T23:26:07.729 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-13T23:26:07.729 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-13T23:26:07.729 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-13T23:26:07.729 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "2/2 daemons upgraded", 2024-09-13T23:26:07.729 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading grafana daemons", 2024-09-13T23:26:07.729 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:26:07.729 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:26:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:08 smithi022 ceph-mon[23758]: from='client.15576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:08 smithi022 ceph-mon[23758]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:08 smithi022 ceph-mon[23758]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:08 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3638388384' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:08.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:08 smithi022 ceph-mon[28440]: from='client.15576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:08.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:08 smithi022 ceph-mon[28440]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:08.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:08 smithi022 ceph-mon[28440]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:08.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:08 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3638388384' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:08.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:08 smithi076 ceph-mon[24587]: from='client.15576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:08.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:08 smithi076 ceph-mon[24587]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:08.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:08 smithi076 ceph-mon[24587]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:08.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:08 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3638388384' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:09.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:09 smithi022 ceph-mon[28440]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:09.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:09 smithi022 ceph-mon[23758]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:09 smithi076 ceph-mon[24587]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:10.595 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:10 smithi022 ceph-mon[23758]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:10.595 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:10 smithi022 ceph-mon[28440]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:10.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:10 smithi076 ceph-mon[24587]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:12.650 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:12 smithi076 ceph-mon[24587]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:12 smithi022 ceph-mon[23758]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:12.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:26:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:26:12] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-09-13T23:26:12.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:12 smithi022 ceph-mon[28440]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:14.261 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:13 smithi076 systemd[1]: Stopping Ceph grafana.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:26:14.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:14 smithi076 ceph-mon[24587]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:14.537 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:14 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[48797]: t=2024-09-13T23:26:14+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-09-13T23:26:14.537 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:14 smithi076 podman[61622]: 2024-09-13 23:26:14.270860453 +0000 UTC m=+0.153438981 container died 969941fc6e23fbe525c67332d529bb587176aeef7b9c656a6ad315cc61c2ea04 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, architecture=x86_64, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, vendor=Red Hat, Inc., description=Ceph Grafana Container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=236.1648460182, io.buildah.version=1.24.2, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, name=ubi8, io.openshift.tags=base rhel8) 2024-09-13T23:26:14.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:14 smithi022 ceph-mon[23758]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:14.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:14 smithi022 ceph-mon[28440]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:14.904 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:14 smithi076 podman[61622]: 2024-09-13 23:26:14.65300883 +0000 UTC m=+0.535587426 container cleanup 969941fc6e23fbe525c67332d529bb587176aeef7b9c656a6ad315cc61c2ea04 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, io.openshift.expose-services=, vcs-type=git, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., version=8.5, name=ubi8, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=236.1648460182, build-date=2022-03-28T10:36:18.413762) 2024-09-13T23:26:14.904 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:14 smithi076 bash[61622]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a 2024-09-13T23:26:15.274 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:14 smithi076 podman[61634]: 2024-09-13 23:26:14.903843459 +0000 UTC m=+0.633919289 container remove 969941fc6e23fbe525c67332d529bb587176aeef7b9c656a6ad315cc61c2ea04 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, distribution-scope=public, io.buildah.version=1.24.2, build-date=2022-03-28T10:36:18.413762, architecture=x86_64, name=ubi8, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.tags=base rhel8, release=236.1648460182, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8) 2024-09-13T23:26:15.274 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-13T23:26:15.274 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 systemd[1]: Stopped Ceph grafana.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:26:15.274 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@grafana.a.service: Consumed 4.944s CPU time. 2024-09-13T23:26:15.274 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 systemd[1]: Starting Ceph grafana.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:26:15.537 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 podman[61765]: 2024-09-13 23:26:15.386758741 +0000 UTC m=+0.016830651 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-09-13T23:26:15.537 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 podman[61765]: 2024-09-13 23:26:15.498619965 +0000 UTC m=+0.128691866 container create 79498011f0b3cf88dd5be1236e6eb6c36b63fa5d8aa49895bb8f691c376f373e (image=quay.io/ceph/grafana:9.4.12, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T23:26:15.870 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 podman[61765]: 2024-09-13 23:26:15.718279861 +0000 UTC m=+0.348351777 container init 79498011f0b3cf88dd5be1236e6eb6c36b63fa5d8aa49895bb8f691c376f373e (image=quay.io/ceph/grafana:9.4.12, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T23:26:15.870 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 podman[61765]: 2024-09-13 23:26:15.721069034 +0000 UTC m=+0.351140940 container start 79498011f0b3cf88dd5be1236e6eb6c36b63fa5d8aa49895bb8f691c376f373e (image=quay.io/ceph/grafana:9.4.12, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T23:26:15.871 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 bash[61765]: 79498011f0b3cf88dd5be1236e6eb6c36b63fa5d8aa49895bb8f691c376f373e 2024-09-13T23:26:15.871 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 systemd[1]: Started Ceph grafana.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:26:16.142 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.869890359Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-09-13T23:26:16.142 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870085139Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T23:26:16.142 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.87014246Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T23:26:16.142 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870151548Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T23:26:16.142 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870156121Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-13T23:26:16.142 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870159262Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-13T23:26:16.142 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870161817Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-13T23:26:16.142 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870164336Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-13T23:26:16.142 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870166836Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-13T23:26:16.142 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870169746Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T23:26:16.143 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870172569Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T23:26:16.143 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870174997Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T23:26:16.143 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870178109Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T23:26:16.143 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870180808Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T23:26:16.143 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870183341Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T23:26:16.143 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870185727Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T23:26:16.143 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870188056Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T23:26:16.143 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870190581Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T23:26:16.143 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=settings t=2024-09-13T23:26:15.870193272Z level=info msg="App mode production" 2024-09-13T23:26:16.143 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=sqlstore t=2024-09-13T23:26:15.870237959Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T23:26:16.144 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=sqlstore t=2024-09-13T23:26:15.870249047Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T23:26:16.144 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.881764171Z level=info msg="Starting DB migrations" 2024-09-13T23:26:16.144 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.885425902Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-13T23:26:16.144 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.896866911Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-13T23:26:16.144 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.898160061Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-13T23:26:16.144 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.898504222Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-13T23:26:16.144 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.899566872Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-13T23:26:16.144 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.900674087Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-13T23:26:16.144 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.901748724Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-13T23:26:16.144 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.902601751Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-13T23:26:16.144 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.903237083Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-13T23:26:16.144 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.904148596Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-13T23:26:16.145 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.905068696Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-13T23:26:16.145 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.907659491Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-13T23:26:16.145 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.91066113Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-13T23:26:16.145 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.913512433Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-13T23:26:16.145 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.916425003Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-13T23:26:16.145 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.919129478Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-13T23:26:16.145 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.921761145Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-13T23:26:16.145 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.924508954Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-13T23:26:16.145 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.927524802Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.930325649Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.933279761Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.936072085Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.938857828Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.941703597Z level=info msg="Executing migration" id="create alert_image table" 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.944607699Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.947377545Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.950284391Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.953086676Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.955683773Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.955772985Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.958261603Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-13T23:26:16.146 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.960855548Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-13T23:26:16.147 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.963594028Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-13T23:26:16.147 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.966399115Z level=info msg="Executing migration" id="create secrets table" 2024-09-13T23:26:16.147 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.969055385Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-13T23:26:16.147 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.975245446Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-13T23:26:16.147 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.978069719Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.980694116Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.98695212Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.993234047Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.995924698Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:15.998697776Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.001487868Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.004119604Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.006903067Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.009673654Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.012407406Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.013204164Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.01400068Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-13T23:26:16.148 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.014416323Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.014878211Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.016003316Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.016836563Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.017616688Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.018488232Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.019314359Z level=info msg="Executing migration" id="add correlation config column" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.020449236Z level=info msg="Executing migration" id="create entity_events table" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.021266007Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.022116385Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.022224977Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.022967461Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.023069982Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T23:26:16.149 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.023806296Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.02462817Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.025411008Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.026300838Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.027176016Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.028064623Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.028915373Z level=info msg="Executing migration" id="Drop public config table" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.029811382Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.030670105Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.03158248Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.032444343Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.033312762Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.039728717Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-13T23:26:16.150 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.041531705Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.043372559Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.044258087Z level=info msg="Executing migration" id="add share column" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.046175168Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.047043532Z level=info msg="Executing migration" id="create file table" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.047893034Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.048929432Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.049586456Z level=info msg="Executing migration" id="create file_meta table" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.050428209Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.053084338Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.055764257Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.05841841Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.061358506Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-13T23:26:16.151 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.064342319Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-13T23:26:16.152 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.067067148Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-13T23:26:16.152 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.069817863Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-13T23:26:16.152 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.070292806Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-13T23:26:16.152 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.071071408Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.071881602Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.072648812Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.073856009Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.075105539Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.075888227Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.089082557Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.089544335Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.090331476Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.09472948Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.095501381Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.09588824Z level=info msg="Executing migration" id="create folder table" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.096638215Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.097128507Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-13T23:26:16.153 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.097598409Z level=info msg="Executing migration" id="Update folder title length" 2024-09-13T23:26:16.154 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.098401721Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-13T23:26:16.154 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=migrator t=2024-09-13T23:26:16.098920319Z level=info msg="migrations completed" performed=105 skipped=376 duration=213.533472ms 2024-09-13T23:26:16.154 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=sqlstore t=2024-09-13T23:26:16.099274696Z level=info msg="Created default organization" 2024-09-13T23:26:16.154 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=plugin.finder t=2024-09-13T23:26:16.140597103Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-13T23:26:16.154 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=secrets t=2024-09-13T23:26:16.140871651Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T23:26:16.154 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=query_data t=2024-09-13T23:26:16.143109032Z level=info msg="Query Service initialization" 2024-09-13T23:26:16.499 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-mon[24587]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:16.499 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-mon[24587]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:16.499 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:16.499 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:16.499 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=live.push_http t=2024-09-13T23:26:16.148316376Z level=info msg="Live Push Gateway initialization" 2024-09-13T23:26:16.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:16 smithi022 ceph-mon[23758]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:16.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:16 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:16.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:16 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:16.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:16 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:16.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:16 smithi022 ceph-mon[28440]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:16.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:16 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:16.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:16 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:16.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:16 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:17.202 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=infra.usagestats.collector t=2024-09-13T23:26:16.796268034Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T23:26:17.203 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=provisioning.datasources t=2024-09-13T23:26:16.798038189Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-13T23:26:17.203 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=provisioning.datasources t=2024-09-13T23:26:16.798048999Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-13T23:26:17.203 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=provisioning.datasources t=2024-09-13T23:26:16.798162554Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T23:26:17.203 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=provisioning.alerting t=2024-09-13T23:26:16.81989884Z level=info msg="starting to provision alerting" 2024-09-13T23:26:17.203 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=provisioning.alerting t=2024-09-13T23:26:16.819911155Z level=info msg="finished to provision alerting" 2024-09-13T23:26:17.203 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=ngalert.state.manager t=2024-09-13T23:26:16.820189216Z level=info msg="Warming state cache for startup" 2024-09-13T23:26:17.203 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=grafanaStorageLogger t=2024-09-13T23:26:16.820397329Z level=info msg="storage starting" 2024-09-13T23:26:17.203 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=http.server t=2024-09-13T23:26:16.822395799Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T23:26:17.203 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=ngalert.state.manager t=2024-09-13T23:26:16.828949855Z level=info msg="State cache has been initialized" states=0 duration=8.760251ms 2024-09-13T23:26:17.203 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=ticker t=2024-09-13T23:26:16.828993908Z level=info msg=starting first_tick=2024-09-13T23:26:20Z 2024-09-13T23:26:17.203 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=ngalert.multiorg.alertmanager t=2024-09-13T23:26:16.829006058Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T23:26:17.203 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=sqlstore.transactions t=2024-09-13T23:26:16.842477363Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T23:26:17.204 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=sqlstore.transactions t=2024-09-13T23:26:16.85520953Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-13T23:26:17.204 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:26:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=sqlstore.transactions t=2024-09-13T23:26:16.866903272Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-09-13T23:26:18.379 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:18 smithi076 ceph-mon[24587]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:18.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:18 smithi022 ceph-mon[23758]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:18.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:18 smithi022 ceph-mon[28440]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:19.161 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:19 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:26:19.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:19 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:26:19.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:19 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:26:20.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:20 smithi022 ceph-mon[23758]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:20.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:20 smithi022 ceph-mon[28440]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:20.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:20 smithi076 ceph-mon[24587]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:21.462 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:21 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:21.462 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:21 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:21.462 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:21 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:21.462 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:21 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:21.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:21 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:21.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:21 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:21.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:21 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:21.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:21 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:21.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:21 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:21.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:21 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:21.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:21 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:21.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:21 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:22.502 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:22 smithi076 ceph-mon[24587]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:22.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:22 smithi022 ceph-mon[23758]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:22.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:26:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:26:22] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.43.0" 2024-09-13T23:26:22.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:22 smithi022 ceph-mon[28440]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:26:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:26:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:26:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:26:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:26:24.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:26:24.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:26:24.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:26:24.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:26:24.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:26:24.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:24 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:26:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:26:24.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:26:24.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:24.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:26:24.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:26:24.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:24.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:26:24.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:26:24.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:26:24.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:26:24.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:24.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:24.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:24.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:26:24.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:26:24.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:24.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:26:24.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:26:24.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:24.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:24.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:26:24.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:26:24.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:26:24.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.685 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.685 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:26:24.685 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:26:24.685 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:26:24.685 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:26:24.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:26:24.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:26:24.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:26:24.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:26:24.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:26:24.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:26:24.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:26:24.687 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:26:24.687 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:26:24.687 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:26:24.687 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.687 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.687 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:26:24.687 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:26:24.687 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:26:24.688 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:26:24.688 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:26:24.688 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:26:24.688 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:26:24.688 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:26:24.689 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:26:24.689 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:26:24.689 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:26:24.689 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:26:24.689 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:26:24.689 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:26:24.689 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:26:24.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:26:24.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:26:24.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:26:24.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:26:24.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:26:24.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:26:24.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.691 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.691 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:26:24.691 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:26:24.691 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:26:24.691 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:26:24.691 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:26:24.691 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:26:24.691 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:26:24.691 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:26:24.691 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:26:24.692 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:26:24.692 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:26:24.692 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.692 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.692 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.692 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.692 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.692 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.692 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.692 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.692 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.693 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.693 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.693 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.693 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:26:24.693 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:26:24.693 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:26:24.693 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:26:24.693 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.694 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:26:24.695 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:26:24.695 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:26:24.695 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:24 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:26:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:25 smithi076 ceph-mon[24587]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:26:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:25 smithi076 ceph-mon[24587]: Upgrade: Finalizing container_image settings 2024-09-13T23:26:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:25 smithi076 ceph-mon[24587]: Upgrade: Complete! 2024-09-13T23:26:25.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:25 smithi022 ceph-mon[23758]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:26:25.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:25 smithi022 ceph-mon[23758]: Upgrade: Finalizing container_image settings 2024-09-13T23:26:25.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:25 smithi022 ceph-mon[23758]: Upgrade: Complete! 2024-09-13T23:26:25.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:25 smithi022 ceph-mon[28440]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T23:26:25.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:25 smithi022 ceph-mon[28440]: Upgrade: Finalizing container_image settings 2024-09-13T23:26:25.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:25 smithi022 ceph-mon[28440]: Upgrade: Complete! 2024-09-13T23:26:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:26 smithi076 ceph-mon[24587]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:26 smithi076 ceph-mon[24587]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:26.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:26 smithi022 ceph-mon[23758]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:26 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:26.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:26 smithi022 ceph-mon[28440]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:26.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:26 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:28.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:28 smithi022 ceph-mon[23758]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:28.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:28 smithi022 ceph-mon[28440]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:28.430 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:28 smithi076 ceph-mon[24587]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:29.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:29 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:29.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:29 smithi022 ceph-mon[23758]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:29.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:29 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:29.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:29 smithi022 ceph-mon[28440]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:29.972 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:29 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:29.972 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:29 smithi076 ceph-mon[24587]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:26:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:26:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:26:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:26:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:31.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:31.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:31.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:26:31.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:26:31.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:31.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:26:31.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:26:31.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:31 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:31.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:31 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:31.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:31 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:31.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:31 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:26:31.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:31 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:26:31.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:31 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:31.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:31 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:26:31.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:31 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:26:31.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:31 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:26:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:32 smithi022 ceph-mon[23758]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:32.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:26:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:26:32] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.43.0" 2024-09-13T23:26:32.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:32 smithi022 ceph-mon[28440]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:32.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:32 smithi076 ceph-mon[24587]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:34.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:34 smithi022 ceph-mon[23758]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:34.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:34 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:26:34.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:34 smithi022 ceph-mon[28440]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:34.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:34 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:26:34.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:34 smithi076 ceph-mon[24587]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:34.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:34 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:26:36.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:36 smithi022 ceph-mon[28440]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:36.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:36 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:36.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:36 smithi022 ceph-mon[23758]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:36.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:36 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:36.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:36 smithi076 ceph-mon[24587]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:36.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:36 smithi076 ceph-mon[24587]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:38.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:38 smithi022 ceph-mon[23758]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:38.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:38 smithi022 ceph-mon[28440]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:38 smithi076 ceph-mon[24587]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:38.844 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-09-13T23:26:39.396 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:39 smithi022 ceph-mon[23758]: from='client.15600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:39.396 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:39 smithi022 ceph-mon[28440]: from='client.15600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:39 smithi076 ceph-mon[24587]: from='client.15600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:40.132 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:40 smithi022 ceph-mon[23758]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:40.133 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:40 smithi022 ceph-mon[28440]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:40.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:40 smithi076 ceph-mon[24587]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:41.083 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:26:41.334 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:41 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1613277016' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:41.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:41 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1613277016' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:41 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/1613277016' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:41.725 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-09-13T23:26:42.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:42 smithi022 ceph-mon[23758]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:42.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:42 smithi022 ceph-mon[28440]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:42.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:42 smithi076 ceph-mon[24587]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:42.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:26:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:26:42] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-09-13T23:26:44.057 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)" 2024-09-13T23:26:44.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:44 smithi022 ceph-mon[23758]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:44.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:44 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2336793371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:44.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:44 smithi022 ceph-mon[28440]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:44.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:44 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2336793371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:44.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:44 smithi076 ceph-mon[24587]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:44.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:44 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2336793371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:44.850 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-09-13T23:26:46.281 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:46 smithi022 ceph-mon[28440]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:46.281 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:46 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:46.281 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:46 smithi022 ceph-mon[23758]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:46.281 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:46 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:46 smithi076 ceph-mon[24587]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:46 smithi076 ceph-mon[24587]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:46.836 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:26:47.415 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:47 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/2678725841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:47.416 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:47 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/2678725841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:47.417 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-09-13T23:26:47.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:47 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/2678725841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:26:48.367 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:48 smithi022 ceph-mon[23758]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:48.367 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:48 smithi022 ceph-mon[28440]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:48 smithi076 ceph-mon[24587]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:49.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:49 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:26:49.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:49 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:26:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:49 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:26:50.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:50 smithi022 ceph-mon[23758]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:50.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:50 smithi022 ceph-mon[28440]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:50.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:50 smithi076 ceph-mon[24587]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:50.773 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:26:51.375 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:51 smithi022 ceph-mon[23758]: from='client.25528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:51.376 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:51 smithi022 ceph-mon[28440]: from='client.25528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:51.404 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T23:26:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:51 smithi076 ceph-mon[24587]: from='client.25528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:52.363 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:52 smithi022 ceph-mon[23758]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:52.363 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:52 smithi022 ceph-mon[28440]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:52 smithi076 ceph-mon[24587]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:52.654 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:26:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:26:52] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.43.0" 2024-09-13T23:26:53.380 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:26:53.380 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": null, 2024-09-13T23:26:53.380 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": false, 2024-09-13T23:26:53.380 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "", 2024-09-13T23:26:53.380 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:26:53.380 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": null, 2024-09-13T23:26:53.380 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-09-13T23:26:53.381 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:26:53.381 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:26:53.962 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T23:26:54.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:54 smithi022 ceph-mon[23758]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:54.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:54 smithi022 ceph-mon[23758]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:54.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:54 smithi022 ceph-mon[28440]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:54.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:54 smithi022 ceph-mon[28440]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:54 smithi076 ceph-mon[24587]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:54 smithi076 ceph-mon[24587]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:26:56.193 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-09-13T23:26:56.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:56 smithi022 ceph-mon[23758]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:56.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:56 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:56 smithi022 ceph-mon[28440]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:56 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:56 smithi076 ceph-mon[24587]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:26:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:56 smithi076 ceph-mon[24587]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:26:56.817 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-09-13T23:26:57.386 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:57 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3880558539' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:26:57.386 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:57 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3880558539' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:26:57.438 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:57 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/3880558539' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:26:58.319 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:26:58 smithi022 ceph-mon[23758]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:58.319 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:26:58 smithi022 ceph-mon[28440]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:26:58.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:26:58 smithi076 ceph-mon[24587]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:00 smithi076 ceph-mon[24587]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:00 smithi076 ceph-mon[24587]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:00 smithi076 ceph-mon[24587]: from='client.25552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mon", "hosts": "smithi076", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:00.539 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:00 smithi022 ceph-mon[28440]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:00.539 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:00 smithi022 ceph-mon[28440]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:00.539 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:00 smithi022 ceph-mon[28440]: from='client.25552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mon", "hosts": "smithi076", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:00.539 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:00 smithi022 ceph-mon[23758]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:00.539 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:00 smithi022 ceph-mon[23758]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:00.539 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:00 smithi022 ceph-mon[23758]: from='client.25552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mon", "hosts": "smithi076", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:00.623 INFO:teuthology.orchestra.run.smithi022.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:01.239 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T23:27:01.816 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:01 smithi022 ceph-mon[23758]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:01.816 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:01 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:01.817 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:01 smithi022 ceph-mon[23758]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:01.817 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:01 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:01.817 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:01 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:01.817 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:01 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:01.817 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:01 smithi022 ceph-mon[28440]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:01.817 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:01 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:01.817 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:01 smithi022 ceph-mon[28440]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:01.818 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:01 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:01.818 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:01 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:01.818 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:01 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:02.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:01 smithi076 ceph-mon[24587]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:02.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:01 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:02.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:01 smithi076 ceph-mon[24587]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:02.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:01 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:02.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:01 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:02.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:01 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:02.577 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:27:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:27:02] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.43.0" 2024-09-13T23:27:03.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:03 smithi022 ceph-mon[23758]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:03.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:03 smithi022 ceph-mon[28440]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:03.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:03 smithi076 ceph-mon[24587]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:04.370 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:27:04.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:04 smithi076 ceph-mon[24587]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:04.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:04.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:04 smithi076 ceph-mon[24587]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:27:04.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:04 smithi076 ceph-mon[24587]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:27:04.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:27:04.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:04.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:04 smithi076 ceph-mon[24587]: Upgrade: Setting container_image for all mgr 2024-09-13T23:27:04.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:04 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:04.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T23:27:04.486 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T23:27:04.487 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:04 smithi076 ceph-mon[24587]: Upgrade: It appears safe to stop mon.b 2024-09-13T23:27:04.487 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:04 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:27:04.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[28440]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[28440]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:27:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[28440]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:27:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:27:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all mgr 2024-09-13T23:27:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T23:27:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T23:27:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[28440]: Upgrade: It appears safe to stop mon.b 2024-09-13T23:27:04.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:27:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[23758]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[23758]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:27:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[23758]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:27:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:27:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all mgr 2024-09-13T23:27:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T23:27:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T23:27:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[23758]: Upgrade: It appears safe to stop mon.b 2024-09-13T23:27:04.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:04 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:27:05.051 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:27:05.051 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (14m) 65s ago 22m 24.7M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:27:05.051 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (49s) 34s ago 22m 72.0M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:27:05.051 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (80s) 65s ago 21m 50.4M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:27:05.051 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (70s) 34s ago 26m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (10m) 65s ago 28m 490M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (28m) 65s ago 28m 75.5M 2048M 17.2.0 e1d6a67b021e 7d59bd9c71e8 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (27m) 34s ago 27m 61.5M 2048M 17.2.0 e1d6a67b021e a1194b4b1c43 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (27m) 65s ago 27m 54.4M 2048M 17.2.0 e1d6a67b021e 336a6f00a2a7 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (14m) 65s ago 22m 16.0M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (13m) 34s ago 22m 16.5M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (26m) 65s ago 26m 52.5M 2993M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (26m) 65s ago 26m 55.4M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (25m) 65s ago 25m 50.7M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (25m) 65s ago 25m 53.1M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (24m) 34s ago 24m 55.8M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (24m) 34s ago 24m 55.9M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:27:05.052 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (24m) 34s ago 24m 54.7M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:27:05.053 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (23m) 34s ago 23m 54.2M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:27:05.053 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (75s) 34s ago 23m 49.6M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:27:05.053 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (21m) 65s ago 21m 76.4M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:27:05.053 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (21m) 34s ago 21m 75.1M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:27:05.369 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:05 smithi022 ceph-mon[23758]: from='client.25558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:05.369 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:05 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:05.369 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:05 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:27:05.369 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:05 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:27:05.369 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:05 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:05.369 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:05 smithi022 ceph-mon[28440]: from='client.25558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:05.369 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:05 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:05.369 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:05 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:27:05.369 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:05 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:27:05.370 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:05 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:05.466 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:27:05.466 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:27:05.466 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T23:27:05.466 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:27:05.466 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:27:05.466 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:27:05.466 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:27:05.466 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:27:05.467 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T23:27:05.467 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:27:05.467 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": {}, 2024-09-13T23:27:05.467 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:27:05.467 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:27:05.467 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:27:05.467 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:27:05.467 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T23:27:05.467 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:27:05.467 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:27:05.467 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:27:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:05 smithi076 ceph-mon[24587]: from='client.25558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:05 smithi076 ceph-mon[24587]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:05 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:27:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:05 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:27:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:05 smithi076 ceph-mon[24587]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:05.816 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:27:05.816 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:27:05.816 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:27:05.816 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi076", 2024-09-13T23:27:05.816 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:27:05.816 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "0/1 daemons upgraded", 2024-09-13T23:27:05.816 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T23:27:05.816 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:27:05.817 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:27:06.500 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:06 smithi076 ceph-mon[24587]: from='client.15660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:06.500 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:06 smithi076 ceph-mon[24587]: Upgrade: Updating mon.b 2024-09-13T23:27:06.500 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:06 smithi076 ceph-mon[24587]: Deploying daemon mon.b on smithi076 2024-09-13T23:27:06.500 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:06 smithi076 ceph-mon[24587]: from='client.25570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:06.501 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:06 smithi076 ceph-mon[24587]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:06.501 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:06 smithi076 ceph-mon[24587]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:06.501 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:06 smithi076 ceph-mon[24587]: from='client.? 172.21.15.22:0/153561697' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:06.501 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:06 smithi076 systemd[1]: Stopping Ceph mon.b for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:27:06.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[23758]: from='client.15660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:06.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[23758]: Upgrade: Updating mon.b 2024-09-13T23:27:06.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[23758]: Deploying daemon mon.b on smithi076 2024-09-13T23:27:06.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[23758]: from='client.25570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:06.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[23758]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:06.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:06.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/153561697' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:06.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[28440]: from='client.15660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:06.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[28440]: Upgrade: Updating mon.b 2024-09-13T23:27:06.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[28440]: Deploying daemon mon.b on smithi076 2024-09-13T23:27:06.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[28440]: from='client.25570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:06.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[28440]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:06.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:06.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:06 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/153561697' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:07.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b[24581]: 2024-09-13T23:27:06.763+0000 7f3ac6214700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:27:07.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b[24581]: 2024-09-13T23:27:06.763+0000 7f3ac6214700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-09-13T23:27:07.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:06 smithi076 podman[63752]: 2024-09-13 23:27:06.795488087 +0000 UTC m=+0.151966841 container died a1194b4b1c438b62e3c161190a0aa082f86579e346d8d623de51f08776e97c85 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., distribution-scope=public, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, version=8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, maintainer=Guillaume Abrioux , release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8) 2024-09-13T23:27:07.442 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:07 smithi076 podman[63752]: 2024-09-13 23:27:07.148145722 +0000 UTC m=+0.504624461 container cleanup a1194b4b1c438b62e3c161190a0aa082f86579e346d8d623de51f08776e97c85 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, version=8, io.k8s.display-name=CentOS Stream 8, release=754, io.openshift.expose-services=, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, ceph=True, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., name=centos-stream) 2024-09-13T23:27:07.442 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:07 smithi076 bash[63752]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b 2024-09-13T23:27:07.783 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:07 smithi076 podman[63764]: 2024-09-13 23:27:07.441999346 +0000 UTC m=+0.649741749 container remove a1194b4b1c438b62e3c161190a0aa082f86579e346d8d623de51f08776e97c85 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b, vendor=Red Hat, Inc., RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, version=8, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, release=754, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, name=centos-stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux ) 2024-09-13T23:27:07.783 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:07 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-13T23:27:07.783 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:07 smithi076 systemd[1]: Stopped Ceph mon.b for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:27:07.783 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:07 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.b.service: Consumed 17.214s CPU time. 2024-09-13T23:27:08.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:07 smithi076 systemd[1]: Starting Ceph mon.b for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:27:08.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:07 smithi076 podman[63894]: 2024-09-13 23:27:07.896495666 +0000 UTC m=+0.016406577 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:08.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 podman[63894]: 2024-09-13 23:27:08.018685558 +0000 UTC m=+0.138596459 container create 65c65bdd515a3c9f75144c876a0851f3f18024755add0f0002d80a37483809ca (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-13T23:27:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 podman[63894]: 2024-09-13 23:27:08.279989528 +0000 UTC m=+0.399900438 container init 65c65bdd515a3c9f75144c876a0851f3f18024755add0f0002d80a37483809ca (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-13T23:27:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 podman[63894]: 2024-09-13 23:27:08.283425663 +0000 UTC m=+0.403336565 container start 65c65bdd515a3c9f75144c876a0851f3f18024755add0f0002d80a37483809ca (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=squid-ecab7be) 2024-09-13T23:27:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T23:27:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable), process ceph-mon, pid 2 2024-09-13T23:27:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: pidfile_write: ignore empty --pid-file 2024-09-13T23:27:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: load: jerasure load: lrc 2024-09-13T23:27:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: RocksDB version: 7.9.2 2024-09-13T23:27:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Git sha 0 2024-09-13T23:27:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Compile date 2024-09-13 16:59:10 2024-09-13T23:27:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: DB SUMMARY 2024-09-13T23:27:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: DB Session ID: CSBI83ETD2AD59193XB2 2024-09-13T23:27:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: CURRENT file: CURRENT 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2814 Bytes 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 322939 ; 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.error_if_exists: 0 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.create_if_missing: 0 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.paranoid_checks: 1 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.env: 0x55f59399e940 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.info_log: 0x55f59426f6e0 2024-09-13T23:27:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T23:27:08.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.statistics: (nil) 2024-09-13T23:27:08.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.use_fsync: 0 2024-09-13T23:27:08.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_log_file_size: 0 2024-09-13T23:27:08.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T23:27:08.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T23:27:08.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T23:27:08.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T23:27:08.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.allow_fallocate: 1 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.use_direct_reads: 0 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.db_log_dir: 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.wal_dir: 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T23:27:08.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T23:27:08.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T23:27:08.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T23:27:08.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.write_buffer_manager: 0x55f59427edc0 2024-09-13T23:27:08.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T23:27:08.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T23:27:08.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T23:27:08.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T23:27:08.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T23:27:08.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.unordered_write: 0 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.row_cache: None 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.wal_filter: None 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.two_write_queues: 0 2024-09-13T23:27:08.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.wal_compression: 0 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.atomic_flush: 0 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.log_readahead_size: 0 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T23:27:08.545 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_background_jobs: 2 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_background_compactions: -1 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_subcompactions: 1 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_open_files: -1 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T23:27:08.546 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T23:27:08.547 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T23:27:08.547 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T23:27:08.547 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_background_flushes: -1 2024-09-13T23:27:08.547 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Compression algorithms supported: 2024-09-13T23:27:08.547 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: kZSTD supported: 0 2024-09-13T23:27:08.547 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: kXpressCompression supported: 0 2024-09-13T23:27:08.547 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: kBZip2Compression supported: 0 2024-09-13T23:27:08.547 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T23:27:08.547 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: kLZ4Compression supported: 1 2024-09-13T23:27:08.547 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: kZlibCompression supported: 1 2024-09-13T23:27:08.547 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T23:27:08.547 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: kSnappyCompression supported: 1 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.merge_operator: 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_filter: None 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f59426f320) 2024-09-13T23:27:08.548 INFO:journalctl@ceph.mon.b.smithi076.stdout: cache_index_and_filter_blocks: 1 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: pin_top_level_index_and_filter: 1 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: index_type: 0 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: data_block_index_type: 0 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: index_shortening: 1 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: checksum: 4 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: no_block_cache: 0 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_cache: 0x55f5942631f0 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_cache_name: BinnedLRUCache 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_cache_options: 2024-09-13T23:27:08.549 INFO:journalctl@ceph.mon.b.smithi076.stdout: capacity : 536870912 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: num_shard_bits : 4 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: strict_capacity_limit : 0 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: high_pri_pool_ratio: 0.000 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_cache_compressed: (nil) 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: persistent_cache: (nil) 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_size: 4096 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_size_deviation: 10 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_restart_interval: 16 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: index_block_restart_interval: 1 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: metadata_block_size: 4096 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: partition_filters: 0 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: use_delta_encoding: 1 2024-09-13T23:27:08.550 INFO:journalctl@ceph.mon.b.smithi076.stdout: filter_policy: bloomfilter 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout: whole_key_filtering: 1 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout: verify_compression: 0 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout: read_amp_bytes_per_bit: 0 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout: format_version: 5 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout: enable_index_compression: 1 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout: block_align: 0 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout: max_auto_readahead_size: 262144 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout: prepopulate_block_cache: 0 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout: initial_auto_readahead_size: 8192 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compression: NoCompression 2024-09-13T23:27:08.551 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.num_levels: 7 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T23:27:08.552 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T23:27:08.553 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T23:27:08.554 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T23:27:08.555 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.inplace_update_support: 0 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.bloom_locality: 0 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.max_successive_merges: 0 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T23:27:08.556 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.ttl: 2592000 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.enable_blob_files: false 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.min_blob_size: 0 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T23:27:08.557 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28965, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f2650d19-b4a6-4dd8-b63c-c06c19f7227d 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726270028310609, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726270028312173, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 211954, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28970, "largest_seqno": 29236, "table_properties": {"data_size": 210050, "index_size": 546, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2902, "raw_average_key_size": 24, "raw_value_size": 207527, "raw_average_value_size": 1743, "num_data_blocks": 24, "num_entries": 119, "num_filter_entries": 119, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726270028, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f2650d19-b4a6-4dd8-b63c-c06c19f7227d", "db_session_id": "CSBI83ETD2AD59193XB2", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726270028312269, "job": 1, "event": "recovery_finished"} 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-09-13T23:27:08.558 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f594290e00 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: DB pointer 0x55f59438c000 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout: ** DB Stats ** 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T23:27:08.559 INFO:journalctl@ceph.mon.b.smithi076.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: L0 1/0 206.99 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 202.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: L6 1/0 8.98 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: Sum 2/0 9.19 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 202.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 202.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T23:27:08.560 INFO:journalctl@ceph.mon.b.smithi076.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 202.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: Cumulative compaction: 0.00 GB write, 52.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: Interval compaction: 0.00 GB write, 52.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: Block cache BinnedLRUCache@0x55f5942631f0#2 capacity: 512.00 MB usage: 1.02 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 7e-06 secs_since: 0 2024-09-13T23:27:08.561 INFO:journalctl@ceph.mon.b.smithi076.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,0.64 KB,0.00012219%) Misc(1,0.00 KB,0%) 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: starting mon.b rank 2 at public addrs [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] at bind addrs [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: mon.b@-1(???) e3 preinit fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: mon.b@-1(???).mds e1 new map 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: mon.b@-1(???).mds e1 print_map 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout: e1 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout: legacy client fscid: -1 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout: 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout: No filesystems configured 2024-09-13T23:27:08.562 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: mon.b@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T23:27:08.563 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T23:27:08.563 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T23:27:08.563 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T23:27:08.563 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 ceph-mon[63913]: mon.b@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-09-13T23:27:08.563 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 bash[63894]: 65c65bdd515a3c9f75144c876a0851f3f18024755add0f0002d80a37483809ca 2024-09-13T23:27:08.563 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:08 smithi076 systemd[1]: Started Ceph mon.b for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:27:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:09 smithi076 ceph-mon[63913]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:09 smithi076 ceph-mon[63913]: mon.b calling monitor election 2024-09-13T23:27:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:09 smithi076 ceph-mon[63913]: mon.a calling monitor election 2024-09-13T23:27:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:09 smithi076 ceph-mon[63913]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T23:27:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:09 smithi076 ceph-mon[63913]: monmap e3: 3 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} 2024-09-13T23:27:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:09 smithi076 ceph-mon[63913]: fsmap 2024-09-13T23:27:09.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:09 smithi076 ceph-mon[63913]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T23:27:09.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:09 smithi076 ceph-mon[63913]: mgrmap e41: y(active, since 96s), standbys: x 2024-09-13T23:27:09.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:09 smithi076 ceph-mon[63913]: overall HEALTH_OK 2024-09-13T23:27:09.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:09 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:09.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:09 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:09.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:09 smithi076 ceph-mon[63913]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:09.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[28440]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:09.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[28440]: mon.b calling monitor election 2024-09-13T23:27:09.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[28440]: mon.a calling monitor election 2024-09-13T23:27:09.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[28440]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T23:27:09.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[28440]: monmap e3: 3 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} 2024-09-13T23:27:09.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[28440]: fsmap 2024-09-13T23:27:09.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[28440]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T23:27:09.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[28440]: mgrmap e41: y(active, since 96s), standbys: x 2024-09-13T23:27:09.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[28440]: overall HEALTH_OK 2024-09-13T23:27:09.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:09.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:09.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[28440]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:09.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[23758]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:09.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[23758]: mon.b calling monitor election 2024-09-13T23:27:09.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[23758]: mon.a calling monitor election 2024-09-13T23:27:09.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[23758]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T23:27:09.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[23758]: monmap e3: 3 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} 2024-09-13T23:27:09.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[23758]: fsmap 2024-09-13T23:27:09.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[23758]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T23:27:09.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[23758]: mgrmap e41: y(active, since 96s), standbys: x 2024-09-13T23:27:09.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[23758]: overall HEALTH_OK 2024-09-13T23:27:09.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:09.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:09.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:09 smithi022 ceph-mon[23758]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:12.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:12 smithi022 ceph-mon[23758]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:12.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:12 smithi022 ceph-mon[28440]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:12 smithi076 ceph-mon[63913]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:12.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:27:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:27:12] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.43.0" 2024-09-13T23:27:13.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:27:13 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:27:13.322+0000 7f7e67b2a640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-13T23:27:13.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:13 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:13.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:13 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:13.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:13 smithi022 ceph-mon[23758]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:13.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:13 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:13.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:13 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:13.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:13 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:13.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:13 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:13.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:13 smithi022 ceph-mon[28440]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:13.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:13 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:13.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:13 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:13.997 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:13 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:13.997 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:13 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:13.997 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:13 smithi076 ceph-mon[63913]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:13.997 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:13 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:13.997 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:13 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:16.404 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:16 smithi076 ceph-mon[63913]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:16.404 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:16 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:16.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:16 smithi022 ceph-mon[23758]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:16 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:16.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:16 smithi022 ceph-mon[28440]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:16.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:16 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:17.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: Detected new or changed devices on smithi076 2024-09-13T23:27:17.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:17.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:17.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:27:17.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T23:27:17.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.445 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: Detected new or changed devices on smithi076 2024-09-13T23:27:17.445 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.445 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.445 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:17.445 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:17.445 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.445 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:27:17.445 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.445 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.445 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.445 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.447 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.448 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.448 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.448 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.448 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.448 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.448 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.448 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.448 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.448 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:27:17.449 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:27:17.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:27:17.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:27:17.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:27:17.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:27:17.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:27:17.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:27:17.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:27:17.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:27:17.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:27:17.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:27:17.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.451 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:27:17.451 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:27:17.451 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:27:17.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: Detected new or changed devices on smithi076 2024-09-13T23:27:17.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:17.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:17.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:27:17.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T23:27:17.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:27:17.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:27:17.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:27:17.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:27:17.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:27:17.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:27:17.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:27:17.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:27:17.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:27:17.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:27:17.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:27:17.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:27:17.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:27:17.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:27:17.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:27:17.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:27:17.796 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:27:17.796 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:27:17.796 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:17.796 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:17.796 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.796 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:17.796 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:17.796 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:17 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:27:17.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:27:17.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:27:17.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:27:17.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:27:17.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:27:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:27:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:27:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:27:17.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:27:17.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:27:17.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:17.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:17.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T23:27:17.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:27:17.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:27:17.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:27:17.932 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.932 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.932 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:27:17.932 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:27:17.932 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:27:17.932 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:27:17.933 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:27:17.933 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:27:17.933 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:27:17.933 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:27:17.933 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:27:17.933 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:27:17.933 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:27:17.933 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:27:17.933 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:27:17.933 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:27:17.933 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.933 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:17.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:17.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:17.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.934 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:27:17.935 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:27:17.935 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:27:17.935 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:27:17.935 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:27:17.935 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:27:17.935 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:27:17.935 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:27:17.935 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:27:17.935 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:27:17.935 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:27:17.935 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.935 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:27:17.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:27:17.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:27:17.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:27:17.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:17.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:17.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:17.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:17.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:17.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:17 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all crash 2024-09-13T23:27:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mds 2024-09-13T23:27:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:27:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T23:27:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:27:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all nfs 2024-09-13T23:27:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:27:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all node-exporter 2024-09-13T23:27:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all prometheus 2024-09-13T23:27:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all alertmanager 2024-09-13T23:27:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all grafana 2024-09-13T23:27:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all loki 2024-09-13T23:27:18.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all promtail 2024-09-13T23:27:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Finalizing container_image settings 2024-09-13T23:27:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: Upgrade: Complete! 2024-09-13T23:27:18.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:18 smithi076 ceph-mon[63913]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:18.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all crash 2024-09-13T23:27:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all mds 2024-09-13T23:27:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:27:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T23:27:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:27:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all nfs 2024-09-13T23:27:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:27:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all node-exporter 2024-09-13T23:27:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all prometheus 2024-09-13T23:27:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all alertmanager 2024-09-13T23:27:18.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all grafana 2024-09-13T23:27:18.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all loki 2024-09-13T23:27:18.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all promtail 2024-09-13T23:27:18.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Finalizing container_image settings 2024-09-13T23:27:18.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: Upgrade: Complete! 2024-09-13T23:27:18.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[28440]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:18.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all crash 2024-09-13T23:27:18.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all mds 2024-09-13T23:27:18.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:27:18.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T23:27:18.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:27:18.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all nfs 2024-09-13T23:27:18.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:27:18.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all node-exporter 2024-09-13T23:27:18.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all prometheus 2024-09-13T23:27:18.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all alertmanager 2024-09-13T23:27:18.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all grafana 2024-09-13T23:27:18.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all loki 2024-09-13T23:27:18.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all promtail 2024-09-13T23:27:18.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Finalizing container_image settings 2024-09-13T23:27:18.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: Upgrade: Complete! 2024-09-13T23:27:18.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:18 smithi022 ceph-mon[23758]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:19.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:19 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:27:19.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:19 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:19.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:19 smithi076 ceph-mon[63913]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:19.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:19 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:27:19.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:19 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:19.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:19 smithi022 ceph-mon[28440]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:19.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:19 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:27:19.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:19 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:19.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:19 smithi022 ceph-mon[23758]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:22.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:22 smithi022 ceph-mon[28440]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:22.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:22 smithi022 ceph-mon[23758]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:22 smithi076 ceph-mon[63913]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:22.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:27:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:27:22] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.43.0" 2024-09-13T23:27:24.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:24 smithi022 ceph-mon[28440]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:24.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:24 smithi022 ceph-mon[23758]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:24 smithi076 ceph-mon[63913]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:26.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:26 smithi022 ceph-mon[28440]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:26.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:26 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:26 smithi022 ceph-mon[23758]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:26 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:26 smithi076 ceph-mon[63913]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:26 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:28.339 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:28 smithi076 ceph-mon[63913]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:28.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:28 smithi022 ceph-mon[23758]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:28.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:28 smithi022 ceph-mon[28440]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:30.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:30 smithi022 ceph-mon[23758]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:30.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:30 smithi022 ceph-mon[28440]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:30.519 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:30 smithi076 ceph-mon[63913]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:32.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:32 smithi022 ceph-mon[23758]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:32.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:32 smithi022 ceph-mon[28440]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:32 smithi076 ceph-mon[63913]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:32.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:27:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:27:32] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.43.0" 2024-09-13T23:27:34.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:34 smithi022 ceph-mon[23758]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:34.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:34 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:27:34.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:34 smithi022 ceph-mon[28440]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:34.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:34 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:27:34.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:34 smithi076 ceph-mon[63913]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:34.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:34 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:27:36.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:36 smithi022 ceph-mon[28440]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:36.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:36 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:36 smithi022 ceph-mon[23758]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:36 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:36 smithi076 ceph-mon[63913]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:36 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:36.925 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:27:37.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:37 smithi022 ceph-mon[23758]: from='client.25588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:37.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:37 smithi022 ceph-mon[28440]: from='client.25588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:37 smithi076 ceph-mon[63913]: from='client.25588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:38.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:38 smithi022 ceph-mon[23758]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:38.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:38 smithi022 ceph-mon[28440]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:38 smithi076 ceph-mon[63913]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:39.170 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:27:39.170 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (14m) 99s ago 22m 24.7M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:27:39.170 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (83s) 26s ago 22m 77.0M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:27:39.170 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (114s) 99s ago 21m 50.4M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:27:39.170 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (104s) 26s ago 27m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:27:39.170 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (11m) 99s ago 29m 490M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:27:39.170 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (29m) 99s ago 29m 75.5M 2048M 17.2.0 e1d6a67b021e 7d59bd9c71e8 2024-09-13T23:27:39.170 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (31s) 26s ago 27m 18.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:27:39.170 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (27m) 99s ago 27m 54.4M 2048M 17.2.0 e1d6a67b021e 336a6f00a2a7 2024-09-13T23:27:39.170 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (14m) 99s ago 23m 16.0M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:27:39.171 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (14m) 26s ago 23m 16.8M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:27:39.171 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (27m) 99s ago 27m 52.5M 2993M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:27:39.171 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (26m) 99s ago 26m 55.4M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:27:39.171 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (26m) 99s ago 26m 50.7M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:27:39.171 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (25m) 99s ago 25m 53.1M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:27:39.171 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (25m) 26s ago 25m 55.9M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:27:39.171 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (25m) 26s ago 25m 55.9M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:27:39.171 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (24m) 26s ago 24m 55.0M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:27:39.171 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (24m) 26s ago 24m 54.3M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:27:39.171 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (109s) 26s ago 23m 60.7M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:27:39.171 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (22m) 99s ago 22m 76.4M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:27:39.171 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (22m) 26s ago 22m 75.7M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:27:39.846 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-09-13T23:27:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:40 smithi076 ceph-mon[63913]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:40 smithi076 ceph-mon[63913]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:40.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:40 smithi022 ceph-mon[28440]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:40.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:40 smithi022 ceph-mon[28440]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:40.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:40 smithi022 ceph-mon[23758]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:40.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:40 smithi022 ceph-mon[23758]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:42.142 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:27:42.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:42 smithi022 ceph-mon[23758]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:42.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:42 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/1543046887' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:42.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:42 smithi022 ceph-mon[28440]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:42.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:42 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/1543046887' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:42.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:42 smithi076 ceph-mon[63913]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:42.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:42 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/1543046887' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:42.744 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:27:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:27:42] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.43.0" 2024-09-13T23:27:42.771 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T23:27:44.283 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:44 smithi022 ceph-mon[23758]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:44.283 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:44 smithi022 ceph-mon[28440]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:44.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:44 smithi076 ceph-mon[63913]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:44.762 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:27:44.762 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": null, 2024-09-13T23:27:44.762 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": false, 2024-09-13T23:27:44.763 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "", 2024-09-13T23:27:44.763 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:27:44.763 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": null, 2024-09-13T23:27:44.763 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-09-13T23:27:44.763 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:27:44.763 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:27:45.401 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T23:27:46.373 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:46 smithi022 ceph-mon[23758]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:46.373 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:46 smithi022 ceph-mon[23758]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:46.373 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:46 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:46.373 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:46 smithi022 ceph-mon[28440]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:46.373 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:46 smithi022 ceph-mon[28440]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:46.373 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:46 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:46 smithi076 ceph-mon[63913]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:46 smithi076 ceph-mon[63913]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:46.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:46 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:47.461 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-09-13T23:27:48.048 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-09-13T23:27:48.358 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:48 smithi022 ceph-mon[28440]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:48.358 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:48 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3083319210' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:27:48.358 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:48 smithi022 ceph-mon[23758]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:48.359 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:48 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3083319210' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:27:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:48 smithi076 ceph-mon[63913]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:48.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:48 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/3083319210' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:27:49.379 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:49 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:27:49.379 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:49 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:27:49.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:49 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:27:50.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:50 smithi022 ceph-mon[23758]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:50.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:50 smithi022 ceph-mon[28440]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:50.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:50 smithi076 ceph-mon[63913]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:51 smithi022 ceph-mon[23758]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:51 smithi022 ceph-mon[28440]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:51 smithi076 ceph-mon[63913]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:51.758 INFO:teuthology.orchestra.run.smithi022.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:52.103 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-mon[23758]: from='client.25624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mon", "hosts": "smithi022", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:52.103 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-mon[23758]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:52.103 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:52.103 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:52.103 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:52.103 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:52.103 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-mon[28440]: from='client.25624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mon", "hosts": "smithi022", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:52.104 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-mon[28440]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:52.104 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:52.104 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:52.104 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:52.104 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:52.409 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T23:27:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:52 smithi076 ceph-mon[63913]: from='client.25624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mon", "hosts": "smithi022", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:52 smithi076 ceph-mon[63913]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:52 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:52 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:52 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:27:52.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:52 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:52.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:27:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:27:52] "GET /metrics HTTP/1.1" 200 37930 "" "Prometheus/2.43.0" 2024-09-13T23:27:53.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:53 smithi022 ceph-mon[23758]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:53.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:53 smithi022 ceph-mon[23758]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:53.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:53 smithi022 ceph-mon[28440]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:53.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:53 smithi022 ceph-mon[28440]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:53.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:53 smithi076 ceph-mon[63913]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:53.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:53 smithi076 ceph-mon[63913]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:54.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[23758]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:54.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:54.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:27:54.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:54.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:54.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T23:27:54.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T23:27:54.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[28440]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:54.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:54.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:27:54.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:54.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:54.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T23:27:54.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:54 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T23:27:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:54 smithi076 ceph-mon[63913]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:27:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:54 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:54 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:27:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:54 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:54.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:54 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:54.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:54 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T23:27:54.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:54 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T23:27:55.296 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:55 smithi022 ceph-mon[28440]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:27:55.297 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:55 smithi022 ceph-mon[28440]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:27:55.297 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:55 smithi022 ceph-mon[28440]: Upgrade: Setting container_image for all mgr 2024-09-13T23:27:55.297 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:55 smithi022 ceph-mon[28440]: Upgrade: It appears safe to stop mon.a 2024-09-13T23:27:55.297 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:55 smithi022 ceph-mon[23758]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:27:55.297 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:55 smithi022 ceph-mon[23758]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:27:55.297 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:55 smithi022 ceph-mon[23758]: Upgrade: Setting container_image for all mgr 2024-09-13T23:27:55.297 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:55 smithi022 ceph-mon[23758]: Upgrade: It appears safe to stop mon.a 2024-09-13T23:27:55.465 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:27:55.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:55 smithi076 ceph-mon[63913]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:27:55.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:55 smithi076 ceph-mon[63913]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:27:55.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:55 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mgr 2024-09-13T23:27:55.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:55 smithi076 ceph-mon[63913]: Upgrade: It appears safe to stop mon.a 2024-09-13T23:27:56.171 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:27:56.171 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (15m) 116s ago 23m 24.7M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:27:56.171 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (100s) 43s ago 22m 77.0M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:27:56.171 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (2m) 116s ago 22m 50.4M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:27:56.171 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (2m) 43s ago 27m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:27:56.171 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (11m) 116s ago 29m 490M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:27:56.171 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (29m) 116s ago 29m 75.5M 2048M 17.2.0 e1d6a67b021e 7d59bd9c71e8 2024-09-13T23:27:56.172 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (48s) 43s ago 27m 18.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:27:56.172 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (28m) 116s ago 28m 54.4M 2048M 17.2.0 e1d6a67b021e 336a6f00a2a7 2024-09-13T23:27:56.172 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (15m) 116s ago 23m 16.0M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:27:56.172 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (14m) 43s ago 23m 16.8M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:27:56.172 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (27m) 116s ago 27m 52.5M 2993M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:27:56.172 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (27m) 116s ago 26m 55.4M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:27:56.172 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (26m) 116s ago 26m 50.7M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:27:56.172 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (26m) 116s ago 26m 53.1M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:27:56.172 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (25m) 43s ago 25m 55.9M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:27:56.172 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (25m) 43s ago 25m 55.9M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:27:56.172 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (25m) 43s ago 25m 55.0M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:27:56.172 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (24m) 43s ago 24m 54.3M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:27:56.173 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (2m) 43s ago 23m 60.7M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:27:56.173 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (22m) 116s ago 22m 76.4M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:27:56.173 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (22m) 43s ago 22m 75.7M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:27:56.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:56 smithi022 ceph-mon[23758]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:56.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:56 smithi022 ceph-mon[23758]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:56 smithi022 ceph-mon[28440]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:56 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:56 smithi076 ceph-mon[63913]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:27:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:56 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:27:56.598 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:27:56.599 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:27:56.599 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-13T23:27:56.599 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T23:27:56.599 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:27:56.599 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:27:56.599 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:27:56.599 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:27:56.599 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:27:56.599 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T23:27:56.599 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:27:56.599 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": {}, 2024-09-13T23:27:56.600 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:27:56.600 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:27:56.600 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:27:56.600 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:27:56.600 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-13T23:27:56.600 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:27:56.600 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:27:56.600 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:27:56.958 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:27:56.958 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:27:56.958 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:27:56.958 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi022", 2024-09-13T23:27:56.958 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:27:56.958 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "0/2 daemons upgraded", 2024-09-13T23:27:56.958 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T23:27:56.959 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:27:56.959 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:27:57.517 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[23758]: from='client.15726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:57.518 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[23758]: from='client.15732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:57.518 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[23758]: from='client.15738 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:57.518 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[23758]: Upgrade: Updating mon.a 2024-09-13T23:27:57.518 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[23758]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:57.518 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:27:57.518 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:27:57.518 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[23758]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:57.518 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[23758]: Deploying daemon mon.a on smithi022 2024-09-13T23:27:57.518 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[23758]: from='client.? 172.21.15.22:0/3079132771' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:57.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[28440]: from='client.15726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:57.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[28440]: from='client.15732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:57.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[28440]: from='client.15738 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:57.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[28440]: Upgrade: Updating mon.a 2024-09-13T23:27:57.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:57.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:27:57.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:27:57.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[28440]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:57.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[28440]: Deploying daemon mon.a on smithi022 2024-09-13T23:27:57.519 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:27:57 smithi022 ceph-mon[28440]: from='client.? 172.21.15.22:0/3079132771' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:57 smithi076 ceph-mon[63913]: from='client.15726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:57 smithi076 ceph-mon[63913]: from='client.15732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:57 smithi076 ceph-mon[63913]: from='client.15738 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:27:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:57 smithi076 ceph-mon[63913]: Upgrade: Updating mon.a 2024-09-13T23:27:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:57 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='mgr.y' 2024-09-13T23:27:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:57 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:27:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:57 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:27:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:57 smithi076 ceph-mon[63913]: from='mgr.25276 172.21.15.22:0/341535964' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:27:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:57 smithi076 ceph-mon[63913]: Deploying daemon mon.a on smithi022 2024-09-13T23:27:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:27:57 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/3079132771' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:27:58.056 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:57 smithi022 systemd[1]: Stopping Ceph mon.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:27:58.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:58 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a[23754]: 2024-09-13T23:27:58.053+0000 7f33b02cb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:27:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:58 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a[23754]: 2024-09-13T23:27:58.053+0000 7f33b02cb700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T23:27:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:58 smithi022 podman[79880]: 2024-09-13 23:27:58.269526681 +0000 UTC m=+0.318317259 container died 7d59bd9c71e84aec45b371b4e13f145608d3160c519df781040f2901358d50c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, version=8, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, maintainer=Guillaume Abrioux , distribution-scope=public, RELEASE=HEAD, vcs-type=git, release=754, io.buildah.version=1.19.8, GIT_CLEAN=True, vendor=Red Hat, Inc.) 2024-09-13T23:27:58.897 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:58 smithi022 podman[79880]: 2024-09-13 23:27:58.6707905 +0000 UTC m=+0.719581107 container cleanup 7d59bd9c71e84aec45b371b4e13f145608d3160c519df781040f2901358d50c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, maintainer=Guillaume Abrioux , name=centos-stream, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, ceph=True, architecture=x86_64, GIT_BRANCH=HEAD, release=754, version=8, RELEASE=HEAD) 2024-09-13T23:27:58.897 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:58 smithi022 bash[79880]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a 2024-09-13T23:27:59.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:58 smithi022 podman[79892]: 2024-09-13 23:27:58.896715764 +0000 UTC m=+0.621606469 container remove 7d59bd9c71e84aec45b371b4e13f145608d3160c519df781040f2901358d50c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, ceph=True) 2024-09-13T23:27:59.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T23:27:59.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 systemd[1]: Stopped Ceph mon.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:27:59.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.a.service: Consumed 27.313s CPU time. 2024-09-13T23:27:59.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 systemd[1]: Starting Ceph mon.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:27:59.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 podman[80020]: 2024-09-13 23:27:59.349530041 +0000 UTC m=+0.016503474 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:27:59.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 podman[80020]: 2024-09-13 23:27:59.447952994 +0000 UTC m=+0.114926430 container create 560b94ba5a9b31b0e9230cce2a7894891f3be7bdeeee8f5da39490d623f2dfc2 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD) 2024-09-13T23:27:59.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 podman[80020]: 2024-09-13 23:27:59.684153117 +0000 UTC m=+0.351126553 container init 560b94ba5a9b31b0e9230cce2a7894891f3be7bdeeee8f5da39490d623f2dfc2 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:27:59.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 podman[80020]: 2024-09-13 23:27:59.687536523 +0000 UTC m=+0.354509960 container start 560b94ba5a9b31b0e9230cce2a7894891f3be7bdeeee8f5da39490d623f2dfc2 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-13T23:27:59.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T23:27:59.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable), process ceph-mon, pid 2 2024-09-13T23:27:59.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: pidfile_write: ignore empty --pid-file 2024-09-13T23:27:59.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: load: jerasure load: lrc 2024-09-13T23:27:59.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: RocksDB version: 7.9.2 2024-09-13T23:27:59.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Git sha 0 2024-09-13T23:27:59.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Compile date 2024-09-13 16:59:10 2024-09-13T23:27:59.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: DB SUMMARY 2024-09-13T23:27:59.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: DB Session ID: PYWSQ519XZ1A2AMHFNHA 2024-09-13T23:27:59.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: CURRENT file: CURRENT 2024-09-13T23:27:59.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T23:27:59.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: MANIFEST file: MANIFEST-000015 size: 2889 Bytes 2024-09-13T23:27:59.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000060.sst 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000058.log size: 1136358 ; 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.error_if_exists: 0 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.create_if_missing: 0 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.paranoid_checks: 1 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.env: 0x557bdfea3940 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.info_log: 0x557be0a4f6e0 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.statistics: (nil) 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.use_fsync: 0 2024-09-13T23:27:59.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_log_file_size: 0 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.allow_fallocate: 1 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.use_direct_reads: 0 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.db_log_dir: 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.wal_dir: 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T23:27:59.930 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.write_buffer_manager: 0x557be0a5edc0 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T23:27:59.931 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T23:27:59.932 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T23:27:59.932 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.unordered_write: 0 2024-09-13T23:27:59.932 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T23:27:59.932 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T23:27:59.932 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T23:27:59.932 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T23:27:59.932 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.row_cache: None 2024-09-13T23:27:59.932 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.wal_filter: None 2024-09-13T23:27:59.932 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T23:27:59.933 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.two_write_queues: 0 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.wal_compression: 0 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.atomic_flush: 0 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.log_readahead_size: 0 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T23:27:59.934 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_background_jobs: 2 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_background_compactions: -1 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_subcompactions: 1 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_open_files: -1 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T23:27:59.935 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_background_flushes: -1 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Compression algorithms supported: 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: kZSTD supported: 0 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: kXpressCompression supported: 0 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: kBZip2Compression supported: 0 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: kLZ4Compression supported: 1 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: kZlibCompression supported: 1 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: kSnappyCompression supported: 1 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-13T23:27:59.936 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.merge_operator: 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_filter: None 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557be0a4f320) 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout: cache_index_and_filter_blocks: 1 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout: pin_top_level_index_and_filter: 1 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout: index_type: 0 2024-09-13T23:27:59.937 INFO:journalctl@ceph.mon.a.smithi022.stdout: data_block_index_type: 0 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: index_shortening: 1 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: checksum: 4 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: no_block_cache: 0 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache: 0x557be0a431f0 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache_name: BinnedLRUCache 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache_options: 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: capacity : 536870912 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: num_shard_bits : 4 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: strict_capacity_limit : 0 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: high_pri_pool_ratio: 0.000 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache_compressed: (nil) 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: persistent_cache: (nil) 2024-09-13T23:27:59.938 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_size: 4096 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_size_deviation: 10 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_restart_interval: 16 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: index_block_restart_interval: 1 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: metadata_block_size: 4096 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: partition_filters: 0 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: use_delta_encoding: 1 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: filter_policy: bloomfilter 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: whole_key_filtering: 1 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: verify_compression: 0 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: read_amp_bytes_per_bit: 0 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: format_version: 5 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: enable_index_compression: 1 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_align: 0 2024-09-13T23:27:59.939 INFO:journalctl@ceph.mon.a.smithi022.stdout: max_auto_readahead_size: 262144 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout: prepopulate_block_cache: 0 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout: initial_auto_readahead_size: 8192 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compression: NoCompression 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.num_levels: 7 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T23:27:59.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T23:27:59.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T23:27:59.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T23:27:59.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.inplace_update_support: 0 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T23:27:59.944 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.bloom_locality: 0 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.max_successive_merges: 0 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.ttl: 2592000 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.enable_blob_files: false 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.min_blob_size: 0 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T23:27:59.945 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 62, last_sequence is 26859, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ef30e49d-f9e9-43be-9d2f-9ecaecbea880 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726270079715194, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726270079720246, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1073249, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 26861, "largest_seqno": 28079, "table_properties": {"data_size": 1068212, "index_size": 2589, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1413, "raw_key_size": 14227, "raw_average_key_size": 25, "raw_value_size": 1056919, "raw_average_value_size": 1900, "num_data_blocks": 116, "num_entries": 556, "num_filter_entries": 556, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726270079, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ef30e49d-f9e9-43be-9d2f-9ecaecbea880", "db_session_id": "PYWSQ519XZ1A2AMHFNHA", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-09-13T23:27:59.946 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726270079720378, "job": 1, "event": "recovery_finished"} 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x557be0a70e00 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: DB pointer 0x557be0b6c000 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** DB Stats ** 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T23:27:59.947 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: L0 1/0 1.02 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 350.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: L6 1/0 8.98 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: Sum 2/0 10.01 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 350.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 350.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T23:27:59.948 INFO:journalctl@ceph.mon.a.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 350.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative compaction: 0.00 GB write, 131.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval compaction: 0.00 GB write, 131.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: Block cache BinnedLRUCache@0x557be0a431f0#2 capacity: 512.00 MB usage: 4.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T23:27:59.949 INFO:journalctl@ceph.mon.a.smithi022.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: starting mon.a rank 0 at public addrs [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] at bind addrs [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: mon.a@-1(???) e3 preinit fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: mon.a@-1(???).mds e1 new map 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: mon.a@-1(???).mds e1 print_map 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout: e1 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout: legacy client fscid: -1 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout: No filesystems configured 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: mon.a@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T23:27:59.950 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T23:27:59.951 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T23:27:59.951 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T23:27:59.951 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 ceph-mon[80039]: mon.a@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-09-13T23:27:59.951 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 bash[80020]: 560b94ba5a9b31b0e9230cce2a7894891f3be7bdeeee8f5da39490d623f2dfc2 2024-09-13T23:27:59.951 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:27:59 smithi022 systemd[1]: Started Ceph mon.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:28:01.518 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: ignoring --setuser ceph since I am not root 2024-09-13T23:28:01.518 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: ignoring --setgroup ceph since I am not root 2024-09-13T23:28:01.518 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mgr[60062]: -- 172.21.15.76:0/2313966890 <== mon.2 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563a02f7d0e0 con 0x563a02f5b400 2024-09-13T23:28:01.518 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:01.310+0000 7f092d775040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T23:28:01.518 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:01.368+0000 7f092d775040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T23:28:01.519 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:01.517+0000 7f092d775040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T23:28:01.519 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:01.519 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: mon.a calling monitor election 2024-09-13T23:28:01.519 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T23:28:01.519 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: monmap epoch 3 2024-09-13T23:28:01.519 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:28:01.519 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: last_changed 2024-09-13T22:59:56.973706+0000 2024-09-13T23:28:01.519 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: created 2024-09-13T22:58:24.106959+0000 2024-09-13T23:28:01.519 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: min_mon_release 17 (quincy) 2024-09-13T23:28:01.519 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: election_strategy: 1 2024-09-13T23:28:01.519 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-13T23:28:01.520 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: 1: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-13T23:28:01.520 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: 2: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.b 2024-09-13T23:28:01.520 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: fsmap 2024-09-13T23:28:01.520 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T23:28:01.520 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: mgrmap e41: y(active, since 2m), standbys: x 2024-09-13T23:28:01.520 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: overall HEALTH_OK 2024-09-13T23:28:01.520 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: from='mgr.25276 ' entity='' 2024-09-13T23:28:01.520 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-mon[63913]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-13T23:28:01.533 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ignoring --setuser ceph since I am not root 2024-09-13T23:28:01.533 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ignoring --setgroup ceph since I am not root 2024-09-13T23:28:01.533 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mgr[61727]: -- 172.21.15.22:0/2917581390 <== mon.0 v2:172.21.15.22:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a338e2b0e0 con 0x55a338e08800 2024-09-13T23:28:01.533 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:01.316+0000 7fd83748b040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T23:28:01.533 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:01.375+0000 7fd83748b040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T23:28:01.533 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:01.531+0000 7fd83748b040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T23:28:01.534 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:01.534 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: mon.a calling monitor election 2024-09-13T23:28:01.534 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T23:28:01.534 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: monmap epoch 3 2024-09-13T23:28:01.534 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:28:01.534 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: last_changed 2024-09-13T22:59:56.973706+0000 2024-09-13T23:28:01.534 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: created 2024-09-13T22:58:24.106959+0000 2024-09-13T23:28:01.534 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: min_mon_release 17 (quincy) 2024-09-13T23:28:01.534 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: election_strategy: 1 2024-09-13T23:28:01.534 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-13T23:28:01.534 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: 1: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-13T23:28:01.535 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: 2: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.b 2024-09-13T23:28:01.535 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: fsmap 2024-09-13T23:28:01.535 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T23:28:01.535 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: mgrmap e41: y(active, since 2m), standbys: x 2024-09-13T23:28:01.535 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: overall HEALTH_OK 2024-09-13T23:28:01.535 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: from='mgr.25276 ' entity='' 2024-09-13T23:28:01.535 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[80039]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-13T23:28:01.535 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:01.535 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: mon.a calling monitor election 2024-09-13T23:28:01.535 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T23:28:01.535 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: monmap epoch 3 2024-09-13T23:28:01.535 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:28:01.536 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: last_changed 2024-09-13T22:59:56.973706+0000 2024-09-13T23:28:01.536 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: created 2024-09-13T22:58:24.106959+0000 2024-09-13T23:28:01.536 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: min_mon_release 17 (quincy) 2024-09-13T23:28:01.536 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: election_strategy: 1 2024-09-13T23:28:01.536 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-13T23:28:01.536 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: 1: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-13T23:28:01.536 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: 2: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.b 2024-09-13T23:28:01.536 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: fsmap 2024-09-13T23:28:01.536 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T23:28:01.537 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: mgrmap e41: y(active, since 2m), standbys: x 2024-09-13T23:28:01.537 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: overall HEALTH_OK 2024-09-13T23:28:01.537 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: from='mgr.25276 ' entity='' 2024-09-13T23:28:01.537 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-mon[28440]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-13T23:28:01.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:01.584+0000 7f092d775040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T23:28:01.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:01.729+0000 7f092d775040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:28:01.902 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:01.600+0000 7fd83748b040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T23:28:01.903 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:01.754+0000 7fd83748b040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:28:01.903 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:01 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:01.901+0000 7fd83748b040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T23:28:02.192 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:01.872+0000 7f092d775040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T23:28:02.491 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:02.232+0000 7fd83748b040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T23:28:02.491 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:02.296+0000 7fd83748b040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T23:28:02.491 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:02.489+0000 7fd83748b040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T23:28:02.531 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:02 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:02.191+0000 7f092d775040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T23:28:02.531 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:02 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:02.253+0000 7f092d775040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T23:28:02.531 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:02 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:02.439+0000 7f092d775040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T23:28:03.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:03 smithi076 ceph-mon[63913]: mgrmap e43: y(active, since 2m), standbys: x 2024-09-13T23:28:03.420 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:03 smithi022 ceph-mon[80039]: mgrmap e43: y(active, since 2m), standbys: x 2024-09-13T23:28:03.420 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:03 smithi022 ceph-mon[28440]: mgrmap e43: y(active, since 2m), standbys: x 2024-09-13T23:28:03.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:03.418+0000 7fd83748b040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T23:28:03.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:03.489+0000 7fd83748b040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T23:28:03.736 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:03.384+0000 7f092d775040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T23:28:03.736 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:03.454+0000 7f092d775040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T23:28:04.453 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:04.189+0000 7f092d775040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T23:28:04.453 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:04.307+0000 7f092d775040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T23:28:04.512 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:04 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:04.238+0000 7fd83748b040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T23:28:04.512 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:04 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:04.360+0000 7fd83748b040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T23:28:04.733 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:04.452+0000 7f092d775040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:28:04.734 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:04.514+0000 7f092d775040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T23:28:04.734 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:04.664+0000 7f092d775040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T23:28:04.734 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:04.732+0000 7f092d775040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T23:28:04.807 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:04 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:04.510+0000 7fd83748b040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T23:28:04.807 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:04 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:04.574+0000 7fd83748b040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T23:28:04.807 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:04 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:04.731+0000 7fd83748b040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T23:28:05.032 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:04.804+0000 7f092d775040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T23:28:05.032 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:04.964+0000 7f092d775040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T23:28:05.105 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:04 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:04.805+0000 7fd83748b040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T23:28:05.105 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:04 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:04.868+0000 7fd83748b040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T23:28:05.105 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:05 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:05.034+0000 7fd83748b040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T23:28:05.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:05 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:05.031+0000 7f092d775040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T23:28:05.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:05 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:05.098+0000 7f092d775040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T23:28:05.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:05 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:05.180+0000 7f092d775040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T23:28:05.287 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:05 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:05.240+0000 7f092d775040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T23:28:05.385 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:05 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:05.102+0000 7fd83748b040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T23:28:05.386 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:05 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:05.172+0000 7fd83748b040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T23:28:05.386 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:05 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:05.259+0000 7fd83748b040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T23:28:05.386 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:05 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:05.322+0000 7fd83748b040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T23:28:05.386 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:05 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:05.383+0000 7fd83748b040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T23:28:05.787 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:05 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:05.300+0000 7f092d775040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T23:28:06.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:06.220+0000 7fd83748b040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T23:28:06.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:28:06] ENGINE Bus STARTING 2024-09-13T23:28:06.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: CherryPy Checker: 2024-09-13T23:28:06.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: The Application mounted at '' has an empty config. 2024-09-13T23:28:06.427 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:06.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-mon[80039]: Standby manager daemon x restarted 2024-09-13T23:28:06.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-mon[80039]: Standby manager daemon x started 2024-09-13T23:28:06.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-mon[80039]: from='mgr.? 172.21.15.76:0/858617959' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:28:06.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-mon[80039]: from='mgr.? 172.21.15.76:0/858617959' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:28:06.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-mon[80039]: from='mgr.? 172.21.15.76:0/858617959' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:28:06.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-mon[80039]: from='mgr.? 172.21.15.76:0/858617959' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:28:06.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-mon[28440]: Standby manager daemon x restarted 2024-09-13T23:28:06.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-mon[28440]: Standby manager daemon x started 2024-09-13T23:28:06.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/858617959' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:28:06.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/858617959' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:28:06.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/858617959' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:28:06.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-mon[28440]: from='mgr.? 172.21.15.76:0/858617959' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:28:06.496 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-mon[63913]: Standby manager daemon x restarted 2024-09-13T23:28:06.496 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-mon[63913]: Standby manager daemon x started 2024-09-13T23:28:06.496 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-mon[63913]: from='mgr.? 172.21.15.76:0/858617959' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T23:28:06.496 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-mon[63913]: from='mgr.? 172.21.15.76:0/858617959' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T23:28:06.496 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-mon[63913]: from='mgr.? 172.21.15.76:0/858617959' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T23:28:06.496 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-mon[63913]: from='mgr.? 172.21.15.76:0/858617959' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T23:28:06.497 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:06.095+0000 7f092d775040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T23:28:06.497 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: [13/Sep/2024:23:28:06] ENGINE Bus STARTING 2024-09-13T23:28:06.497 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: CherryPy Checker: 2024-09-13T23:28:06.497 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: The Application mounted at '' has an empty config. 2024-09-13T23:28:06.497 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: 2024-09-13T23:28:06.497 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: [13/Sep/2024:23:28:06] ENGINE Serving on http://:::9283 2024-09-13T23:28:06.497 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:28:06 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x[60058]: [13/Sep/2024:23:28:06] ENGINE Bus STARTED 2024-09-13T23:28:06.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:28:06] ENGINE Serving on http://:::9283 2024-09-13T23:28:06.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:06 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:28:06] ENGINE Bus STARTED 2024-09-13T23:28:07.426 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:07.237+0000 7fd7d6194640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-13T23:28:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: mgrmap e44: y(active, since 2m), standbys: x 2024-09-13T23:28:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: Active manager daemon y restarted 2024-09-13T23:28:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: Activating manager daemon y 2024-09-13T23:28:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T23:28:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: mgrmap e45: y(active, starting, since 0.00864516s), standbys: x 2024-09-13T23:28:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:28:07.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:28:07.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:28:07.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:28:07.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: Manager daemon y is now available 2024-09-13T23:28:07.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:28:07.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:28:07.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:28:07.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: mgrmap e44: y(active, since 2m), standbys: x 2024-09-13T23:28:07.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: Active manager daemon y restarted 2024-09-13T23:28:07.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: Activating manager daemon y 2024-09-13T23:28:07.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T23:28:07.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: mgrmap e45: y(active, starting, since 0.00864516s), standbys: x 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:28:07.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:28:07.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:28:07.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:28:07.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: Manager daemon y is now available 2024-09-13T23:28:07.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:28:07.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:28:07.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:28:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: mgrmap e44: y(active, since 2m), standbys: x 2024-09-13T23:28:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: Active manager daemon y restarted 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: Activating manager daemon y 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: mgrmap e45: y(active, starting, since 0.00864516s), standbys: x 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:28:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:28:07.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:28:07.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:28:07.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:28:07.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T23:28:07.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T23:28:07.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T23:28:07.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: Manager daemon y is now available 2024-09-13T23:28:07.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:28:07.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T23:28:07.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T23:28:08.296 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:08 smithi022 ceph-mon[80039]: mgrmap e46: y(active, since 1.00963s), standbys: x 2024-09-13T23:28:08.296 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:08 smithi022 ceph-mon[80039]: [13/Sep/2024:23:28:07] ENGINE Bus STARTING 2024-09-13T23:28:08.296 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:08 smithi022 ceph-mon[80039]: [13/Sep/2024:23:28:07] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:28:08.296 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:08 smithi022 ceph-mon[80039]: [13/Sep/2024:23:28:07] ENGINE Error in HTTPServer.serve 2024-09-13T23:28:08.296 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:28:08.296 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:28:08.296 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:28:08.296 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:28:08.296 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._run(expiration_interval) 2024-09-13T23:28:08.296 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout: s = self.context.wrap_socket( 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.sslsocket_class._create( 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.do_handshake() 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._sslobj.do_handshake() 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:28:08.297 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:08 smithi022 ceph-mon[80039]: [13/Sep/2024:23:28:08] ENGINE Serving on http://172.21.15.22:8765 2024-09-13T23:28:08.298 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:08 smithi022 ceph-mon[80039]: [13/Sep/2024:23:28:08] ENGINE Bus STARTED 2024-09-13T23:28:08.298 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:08 smithi022 ceph-mon[28440]: mgrmap e46: y(active, since 1.00963s), standbys: x 2024-09-13T23:28:08.298 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:08 smithi022 ceph-mon[28440]: [13/Sep/2024:23:28:07] ENGINE Bus STARTING 2024-09-13T23:28:08.298 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:08 smithi022 ceph-mon[28440]: [13/Sep/2024:23:28:07] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:28:08.298 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:08 smithi022 ceph-mon[28440]: [13/Sep/2024:23:28:07] ENGINE Error in HTTPServer.serve 2024-09-13T23:28:08.298 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-13T23:28:08.298 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:28:08.298 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:28:08.298 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:28:08.298 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._run(expiration_interval) 2024-09-13T23:28:08.299 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:28:08.299 INFO:journalctl@ceph.mon.c.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:28:08.299 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:28:08.299 INFO:journalctl@ceph.mon.c.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:28:08.299 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:28:08.299 INFO:journalctl@ceph.mon.c.smithi022.stdout: s = self.context.wrap_socket( 2024-09-13T23:28:08.299 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:28:08.299 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.sslsocket_class._create( 2024-09-13T23:28:08.299 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:28:08.299 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.do_handshake() 2024-09-13T23:28:08.299 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:28:08.299 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._sslobj.do_handshake() 2024-09-13T23:28:08.300 INFO:journalctl@ceph.mon.c.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:28:08.300 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:08 smithi022 ceph-mon[28440]: [13/Sep/2024:23:28:08] ENGINE Serving on http://172.21.15.22:8765 2024-09-13T23:28:08.300 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:08 smithi022 ceph-mon[28440]: [13/Sep/2024:23:28:08] ENGINE Bus STARTED 2024-09-13T23:28:08.300 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:07.990+0000 7fd793cc8640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:23:28:07] ENGINE Error in HTTPServer.serve 2024-09-13T23:28:08.300 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: Traceback (most recent call last): 2024-09-13T23:28:08.300 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:28:08.300 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: self._connections.run(self.expiration_interval) 2024-09-13T23:28:08.300 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:28:08.300 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: self._run(expiration_interval) 2024-09-13T23:28:08.300 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:28:08.300 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:28:08.301 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:28:08.301 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:28:08.301 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:28:08.301 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: s = self.context.wrap_socket( 2024-09-13T23:28:08.301 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:28:08.301 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: return self.sslsocket_class._create( 2024-09-13T23:28:08.301 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:28:08.301 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: self.do_handshake() 2024-09-13T23:28:08.301 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:28:08.301 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: self._sslobj.do_handshake() 2024-09-13T23:28:08.301 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:28:08.301 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:08 smithi076 ceph-mon[63913]: mgrmap e46: y(active, since 1.00963s), standbys: x 2024-09-13T23:28:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:08 smithi076 ceph-mon[63913]: [13/Sep/2024:23:28:07] ENGINE Bus STARTING 2024-09-13T23:28:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:08 smithi076 ceph-mon[63913]: [13/Sep/2024:23:28:07] ENGINE Serving on https://172.21.15.22:7150 2024-09-13T23:28:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:08 smithi076 ceph-mon[63913]: [13/Sep/2024:23:28:07] ENGINE Error in HTTPServer.serve 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: Traceback (most recent call last): 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._run(expiration_interval) 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: s = self.context.wrap_socket( 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T23:28:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout: return self.sslsocket_class._create( 2024-09-13T23:28:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T23:28:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: self.do_handshake() 2024-09-13T23:28:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T23:28:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: self._sslobj.do_handshake() 2024-09-13T23:28:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T23:28:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:08 smithi076 ceph-mon[63913]: [13/Sep/2024:23:28:08] ENGINE Serving on http://172.21.15.22:8765 2024-09-13T23:28:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:08 smithi076 ceph-mon[63913]: [13/Sep/2024:23:28:08] ENGINE Bus STARTED 2024-09-13T23:28:09.331 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:09 smithi076 ceph-mon[63913]: mgrmap e47: y(active, since 2s), standbys: x 2024-09-13T23:28:09.332 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:09 smithi076 ceph-mon[63913]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:28:09.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:09 smithi022 ceph-mon[28440]: mgrmap e47: y(active, since 2s), standbys: x 2024-09-13T23:28:09.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:09 smithi022 ceph-mon[28440]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:28:09.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:09 smithi022 ceph-mon[80039]: mgrmap e47: y(active, since 2s), standbys: x 2024-09-13T23:28:09.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:09 smithi022 ceph-mon[80039]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:28:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:11 smithi076 ceph-mon[63913]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:28:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:11 smithi076 ceph-mon[63913]: mgrmap e48: y(active, since 4s), standbys: x 2024-09-13T23:28:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:11.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:11 smithi022 ceph-mon[80039]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:28:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:11 smithi022 ceph-mon[80039]: mgrmap e48: y(active, since 4s), standbys: x 2024-09-13T23:28:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:11 smithi022 ceph-mon[28440]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:28:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:11 smithi022 ceph-mon[28440]: mgrmap e48: y(active, since 4s), standbys: x 2024-09-13T23:28:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:11 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:11 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:12 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:12 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:12 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:12 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:12 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:12 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:28:12] "GET /metrics HTTP/1.1" 200 35128 "" "Prometheus/2.43.0" 2024-09-13T23:28:12.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:12.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:12 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:13.705 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:13 smithi076 ceph-mon[63913]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:28:13.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:13 smithi022 ceph-mon[80039]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:28:13.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:13 smithi022 ceph-mon[28440]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:28:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:16 smithi076 ceph-mon[63913]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T23:28:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:16.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:28:16.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:16 smithi076 ceph-mon[63913]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:28:16.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:16 smithi076 ceph-mon[63913]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:28:16.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[80039]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T23:28:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:28:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[80039]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:28:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[80039]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:28:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[28440]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T23:28:16.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:16.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:28:16.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-13T23:28:16.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-13T23:28:16.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:16.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:16 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:17.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:17 smithi076 ceph-mon[63913]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:28:17.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:17 smithi076 ceph-mon[63913]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:28:17.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:17 smithi076 ceph-mon[63913]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:28:17.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:17 smithi076 ceph-mon[63913]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:28:17.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:17 smithi076 ceph-mon[63913]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:28:17.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:17 smithi076 ceph-mon[63913]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:28:17.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:17 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:28:17.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:17 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:17.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:17 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T23:28:17.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:17 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T23:28:17.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:17 smithi076 ceph-mon[63913]: Upgrade: It appears safe to stop mon.c 2024-09-13T23:28:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[28440]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:28:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[28440]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:28:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[28440]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:28:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[28440]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:28:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[28440]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:28:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[28440]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:28:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:28:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:17.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T23:28:17.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T23:28:17.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[28440]: Upgrade: It appears safe to stop mon.c 2024-09-13T23:28:17.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[80039]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:28:17.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[80039]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T23:28:17.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[80039]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:28:17.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[80039]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-13T23:28:17.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[80039]: Updating smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:28:17.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[80039]: Updating smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T23:28:17.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:28:17.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:17.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T23:28:17.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T23:28:17.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:17 smithi022 ceph-mon[80039]: Upgrade: It appears safe to stop mon.c 2024-09-13T23:28:18.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:18 smithi022 ceph-mon[80039]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T23:28:18.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:18 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:18.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:18 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:28:18.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:18 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:28:18.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:18 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:18.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:18 smithi022 ceph-mon[28440]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T23:28:18.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:18 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:18.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:18 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:28:18.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:18 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:28:18.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:18 smithi022 ceph-mon[28440]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:18 smithi076 ceph-mon[63913]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T23:28:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:18 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:18 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:28:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:18 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:28:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:18 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:19.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:19 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:19.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:19 smithi022 ceph-mon[80039]: Upgrade: Updating mon.c 2024-09-13T23:28:19.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:19 smithi022 ceph-mon[80039]: Deploying daemon mon.c on smithi022 2024-09-13T23:28:19.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:19 smithi022 ceph-mon[28440]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:19.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:19 smithi022 ceph-mon[28440]: Upgrade: Updating mon.c 2024-09-13T23:28:19.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:19 smithi022 ceph-mon[28440]: Deploying daemon mon.c on smithi022 2024-09-13T23:28:19.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:19 smithi022 systemd[1]: Stopping Ceph mon.c for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:28:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:19 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:19 smithi076 ceph-mon[63913]: Upgrade: Updating mon.c 2024-09-13T23:28:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:19 smithi076 ceph-mon[63913]: Deploying daemon mon.c on smithi022 2024-09-13T23:28:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c[28436]: 2024-09-13T23:28:19.339+0000 7f9ab8b87700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:28:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:19 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c[28436]: 2024-09-13T23:28:19.339+0000 7f9ab8b87700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-09-13T23:28:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:19 smithi022 podman[82232]: 2024-09-13 23:28:19.514363798 +0000 UTC m=+0.296145218 container died 336a6f00a2a741e6666e2d455962c0dc9270df596d3344eb03571323923cea58 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c, version=8, release=754, io.buildah.version=1.19.8, distribution-scope=public, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, ceph=True, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., vcs-type=git, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux ) 2024-09-13T23:28:20.175 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:19 smithi022 podman[82232]: 2024-09-13 23:28:19.931678601 +0000 UTC m=+0.713460024 container cleanup 336a6f00a2a741e6666e2d455962c0dc9270df596d3344eb03571323923cea58 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , version=8, release=754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, vcs-type=git, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, vendor=Red Hat, Inc., GIT_CLEAN=True, architecture=x86_64) 2024-09-13T23:28:20.175 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:19 smithi022 bash[82232]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c 2024-09-13T23:28:20.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 podman[82247]: 2024-09-13 23:28:20.174598036 +0000 UTC m=+0.650792082 container remove 336a6f00a2a741e6666e2d455962c0dc9270df596d3344eb03571323923cea58 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, distribution-scope=public, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , RELEASE=HEAD, architecture=x86_64, io.buildah.version=1.19.8, vcs-type=git, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, vendor=Red Hat, Inc., version=8) 2024-09-13T23:28:20.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-13T23:28:20.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 systemd[1]: Stopped Ceph mon.c for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:28:20.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.c.service: Consumed 16.314s CPU time. 2024-09-13T23:28:20.850 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 systemd[1]: Starting Ceph mon.c for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:28:20.850 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 podman[82377]: 2024-09-13 23:28:20.725584192 +0000 UTC m=+0.107335849 container create daaa9197af0446babb1008e27a539b44470d26f6fe53e571db500a6533ceee69 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be) 2024-09-13T23:28:20.850 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 podman[82377]: 2024-09-13 23:28:20.634259055 +0000 UTC m=+0.016010678 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:28:21.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 podman[82377]: 2024-09-13 23:28:20.969653642 +0000 UTC m=+0.351405257 container init daaa9197af0446babb1008e27a539b44470d26f6fe53e571db500a6533ceee69 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-13T23:28:21.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 podman[82377]: 2024-09-13 23:28:20.973166093 +0000 UTC m=+0.354917714 container start daaa9197af0446babb1008e27a539b44470d26f6fe53e571db500a6533ceee69 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:28:21.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T23:28:21.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable), process ceph-mon, pid 2 2024-09-13T23:28:21.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: pidfile_write: ignore empty --pid-file 2024-09-13T23:28:21.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: load: jerasure load: lrc 2024-09-13T23:28:21.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: RocksDB version: 7.9.2 2024-09-13T23:28:21.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Git sha 0 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Compile date 2024-09-13 16:59:10 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: DB SUMMARY 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: DB Session ID: KZKR60B19MMZOM4UDUNT 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: CURRENT file: CURRENT 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3004 Bytes 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000057.sst 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000055.log size: 2596191 ; 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.error_if_exists: 0 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.create_if_missing: 0 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.paranoid_checks: 1 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T23:28:21.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T23:28:21.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T23:28:21.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.env: 0x56196e905940 2024-09-13T23:28:21.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T23:28:21.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.info_log: 0x56196f5a36e0 2024-09-13T23:28:21.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T23:28:21.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.statistics: (nil) 2024-09-13T23:28:21.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.use_fsync: 0 2024-09-13T23:28:21.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.max_log_file_size: 0 2024-09-13T23:28:21.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T23:28:21.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T23:28:21.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T23:28:21.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T23:28:21.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.allow_fallocate: 1 2024-09-13T23:28:21.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T23:28:21.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T23:28:21.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.use_direct_reads: 0 2024-09-13T23:28:21.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T23:28:21.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T23:28:21.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.db_log_dir: 2024-09-13T23:28:21.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.wal_dir: 2024-09-13T23:28:21.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T23:28:21.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T23:28:21.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T23:28:21.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T23:28:21.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T23:28:21.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T23:28:21.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T23:28:21.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T23:28:21.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.write_buffer_manager: 0x56196f5b2dc0 2024-09-13T23:28:21.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T23:28:21.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T23:28:21.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.unordered_write: 0 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.row_cache: None 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.wal_filter: None 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T23:28:21.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.two_write_queues: 0 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.wal_compression: 0 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.atomic_flush: 0 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.log_readahead_size: 0 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T23:28:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.max_background_jobs: 2 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.max_background_compactions: -1 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.max_subcompactions: 1 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T23:28:21.185 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T23:28:21.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.max_open_files: -1 2024-09-13T23:28:21.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T23:28:21.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T23:28:21.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T23:28:21.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T23:28:21.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Options.max_background_flushes: -1 2024-09-13T23:28:21.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Compression algorithms supported: 2024-09-13T23:28:21.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: kZSTD supported: 0 2024-09-13T23:28:21.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: kXpressCompression supported: 0 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: kBZip2Compression supported: 0 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: kLZ4Compression supported: 1 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: kZlibCompression supported: 1 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: kSnappyCompression supported: 1 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:20 smithi022 ceph-mon[82397]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.merge_operator: 2024-09-13T23:28:21.187 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_filter: None 2024-09-13T23:28:21.188 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T23:28:21.188 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T23:28:21.188 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T23:28:21.188 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T23:28:21.188 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56196f5a3320) 2024-09-13T23:28:21.188 INFO:journalctl@ceph.mon.c.smithi022.stdout: cache_index_and_filter_blocks: 1 2024-09-13T23:28:21.188 INFO:journalctl@ceph.mon.c.smithi022.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T23:28:21.188 INFO:journalctl@ceph.mon.c.smithi022.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T23:28:21.188 INFO:journalctl@ceph.mon.c.smithi022.stdout: pin_top_level_index_and_filter: 1 2024-09-13T23:28:21.188 INFO:journalctl@ceph.mon.c.smithi022.stdout: index_type: 0 2024-09-13T23:28:21.189 INFO:journalctl@ceph.mon.c.smithi022.stdout: data_block_index_type: 0 2024-09-13T23:28:21.189 INFO:journalctl@ceph.mon.c.smithi022.stdout: index_shortening: 1 2024-09-13T23:28:21.189 INFO:journalctl@ceph.mon.c.smithi022.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T23:28:21.189 INFO:journalctl@ceph.mon.c.smithi022.stdout: checksum: 4 2024-09-13T23:28:21.189 INFO:journalctl@ceph.mon.c.smithi022.stdout: no_block_cache: 0 2024-09-13T23:28:21.189 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache: 0x56196f5971f0 2024-09-13T23:28:21.189 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache_name: BinnedLRUCache 2024-09-13T23:28:21.189 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache_options: 2024-09-13T23:28:21.189 INFO:journalctl@ceph.mon.c.smithi022.stdout: capacity : 536870912 2024-09-13T23:28:21.189 INFO:journalctl@ceph.mon.c.smithi022.stdout: num_shard_bits : 4 2024-09-13T23:28:21.189 INFO:journalctl@ceph.mon.c.smithi022.stdout: strict_capacity_limit : 0 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: high_pri_pool_ratio: 0.000 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache_compressed: (nil) 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: persistent_cache: (nil) 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_size: 4096 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_size_deviation: 10 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_restart_interval: 16 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: index_block_restart_interval: 1 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: metadata_block_size: 4096 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: partition_filters: 0 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: use_delta_encoding: 1 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: filter_policy: bloomfilter 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: whole_key_filtering: 1 2024-09-13T23:28:21.190 INFO:journalctl@ceph.mon.c.smithi022.stdout: verify_compression: 0 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout: read_amp_bytes_per_bit: 0 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout: format_version: 5 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout: enable_index_compression: 1 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_align: 0 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout: max_auto_readahead_size: 262144 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout: prepopulate_block_cache: 0 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout: initial_auto_readahead_size: 8192 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compression: NoCompression 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T23:28:21.191 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T23:28:21.192 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T23:28:21.192 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.num_levels: 7 2024-09-13T23:28:21.192 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T23:28:21.192 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T23:28:21.192 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T23:28:21.192 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T23:28:21.192 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T23:28:21.192 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T23:28:21.192 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T23:28:21.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T23:28:21.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T23:28:21.195 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.inplace_update_support: 0 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T23:28:21.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T23:28:21.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.bloom_locality: 0 2024-09-13T23:28:21.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.max_successive_merges: 0 2024-09-13T23:28:21.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T23:28:21.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T23:28:21.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T23:28:21.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T23:28:21.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.ttl: 2592000 2024-09-13T23:28:21.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T23:28:21.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T23:28:21.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T23:28:21.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.enable_blob_files: false 2024-09-13T23:28:21.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.min_blob_size: 0 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 57.sst 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 59, last_sequence is 30851, log_number is 55,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 55 2024-09-13T23:28:21.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 99636e42-c187-4cb6-a3de-3fcabc194a01 2024-09-13T23:28:21.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726270101000442, "job": 1, "event": "recovery_started", "wal_files": [55]} 2024-09-13T23:28:21.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #55 mode 2 2024-09-13T23:28:21.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726270101007481, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 60, "file_size": 1663054, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 30857, "largest_seqno": 31248, "table_properties": {"data_size": 1660134, "index_size": 1368, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 4508, "raw_average_key_size": 24, "raw_value_size": 1655815, "raw_average_value_size": 8854, "num_data_blocks": 61, "num_entries": 187, "num_filter_entries": 187, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726270101, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "99636e42-c187-4cb6-a3de-3fcabc194a01", "db_session_id": "KZKR60B19MMZOM4UDUNT", "orig_file_number": 60, "seqno_to_time_mapping": "N/A"}} 2024-09-13T23:28:21.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726270101007601, "job": 1, "event": "recovery_finished"} 2024-09-13T23:28:21.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [db/version_set.cc:5047] Creating manifest 62 2024-09-13T23:28:21.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T23:28:21.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000055.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T23:28:21.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56196f5c4e00 2024-09-13T23:28:21.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: DB pointer 0x56196f6c0000 2024-09-13T23:28:21.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T23:28:21.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T23:28:21.199 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** DB Stats ** 2024-09-13T23:28:21.200 INFO:journalctl@ceph.mon.c.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T23:28:21.200 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T23:28:21.200 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T23:28:21.200 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T23:28:21.200 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T23:28:21.200 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T23:28:21.200 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T23:28:21.200 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T23:28:21.200 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-13T23:28:21.200 INFO:journalctl@ceph.mon.c.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T23:28:21.200 INFO:journalctl@ceph.mon.c.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T23:28:21.200 INFO:journalctl@ceph.mon.c.smithi022.stdout: L0 1/0 1.59 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 490.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: L6 1/0 10.39 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: Sum 2/0 11.98 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 490.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 490.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 490.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T23:28:21.201 INFO:journalctl@ceph.mon.c.smithi022.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-13T23:28:21.202 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T23:28:21.202 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T23:28:21.202 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T23:28:21.202 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T23:28:21.202 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative compaction: 0.00 GB write, 166.23 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T23:28:21.202 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval compaction: 0.00 GB write, 166.23 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T23:28:21.202 INFO:journalctl@ceph.mon.c.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T23:28:21.202 INFO:journalctl@ceph.mon.c.smithi022.stdout: Block cache BinnedLRUCache@0x56196f5971f0#2 capacity: 512.00 MB usage: 2.05 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.7e-05 secs_since: 0 2024-09-13T23:28:21.202 INFO:journalctl@ceph.mon.c.smithi022.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-09-13T23:28:21.202 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T23:28:21.202 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T23:28:21.202 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: starting mon.c rank 1 at public addrs [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] at bind addrs [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: mon.c@-1(???) e3 preinit fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: mon.c@-1(???).mds e1 new map 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: mon.c@-1(???).mds e1 print_map 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout: e1 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout: legacy client fscid: -1 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout: No filesystems configured 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: mon.c@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T23:28:21.203 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T23:28:21.204 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T23:28:21.204 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: mon.c@-1(???).paxosservice(auth 1..28) refresh upgraded, format 0 -> 3 2024-09-13T23:28:21.204 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 bash[82377]: daaa9197af0446babb1008e27a539b44470d26f6fe53e571db500a6533ceee69 2024-09-13T23:28:21.204 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 systemd[1]: Started Ceph mon.c for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:28:21.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:21 smithi076 ceph-mon[63913]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T23:28:21.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[80039]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T23:28:21.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:21 smithi022 ceph-mon[82397]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T23:28:22.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:28:22] "GET /metrics HTTP/1.1" 200 37867 "" "Prometheus/2.43.0" 2024-09-13T23:28:26.426 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:28:26.017+0000 7fd7d6194640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-13T23:28:26.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:28:26.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: mon.a calling monitor election 2024-09-13T23:28:26.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: mon.c calling monitor election 2024-09-13T23:28:26.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:28:26.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:28:26.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:28:26.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:28:26.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T23:28:26.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: monmap epoch 4 2024-09-13T23:28:26.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:28:26.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: last_changed 2024-09-13T23:28:21.450332+0000 2024-09-13T23:28:26.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: created 2024-09-13T22:58:24.106959+0000 2024-09-13T23:28:26.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: min_mon_release 19 (squid) 2024-09-13T23:28:26.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: election_strategy: 1 2024-09-13T23:28:26.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-13T23:28:26.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: 1: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-13T23:28:26.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: 2: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.b 2024-09-13T23:28:26.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: fsmap 2024-09-13T23:28:26.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T23:28:26.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: mgrmap e48: y(active, since 20s), standbys: x 2024-09-13T23:28:26.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: overall HEALTH_OK 2024-09-13T23:28:26.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:26.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:26.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:28:26.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: mon.a calling monitor election 2024-09-13T23:28:26.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: mon.c calling monitor election 2024-09-13T23:28:26.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:28:26.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:28:26.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:28:26.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:28:26.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T23:28:26.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: monmap epoch 4 2024-09-13T23:28:26.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:28:26.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: last_changed 2024-09-13T23:28:21.450332+0000 2024-09-13T23:28:26.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: created 2024-09-13T22:58:24.106959+0000 2024-09-13T23:28:26.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: min_mon_release 19 (squid) 2024-09-13T23:28:26.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: election_strategy: 1 2024-09-13T23:28:26.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-13T23:28:26.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: 1: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-13T23:28:26.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: 2: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.b 2024-09-13T23:28:26.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: fsmap 2024-09-13T23:28:26.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T23:28:26.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: mgrmap e48: y(active, since 20s), standbys: x 2024-09-13T23:28:26.930 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: overall HEALTH_OK 2024-09-13T23:28:26.931 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:26.931 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:26 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:27.330 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:28:27.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:27 smithi022 ceph-mon[80039]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:27.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:27 smithi022 ceph-mon[82397]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:28.037 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:28:28.037 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (15m) 16s ago 23m 24.4M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:28:28.037 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (2m) 17s ago 23m 82.8M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:28:28.037 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (2m) 16s ago 22m 51.5M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:28:28.037 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (2m) 17s ago 28m 437M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:28:28.037 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (12m) 16s ago 29m 473M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:28:28.037 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (28s) 16s ago 30m 30.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:28:28.037 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (80s) 17s ago 28m 35.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 starting - - - 2048M 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (15m) 16s ago 24m 16.4M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (15m) 17s ago 24m 16.1M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (27m) 16s ago 27m 52.5M 2993M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (27m) 16s ago 27m 55.5M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (27m) 16s ago 27m 51.0M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (26m) 16s ago 26m 53.1M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (26m) 17s ago 26m 56.1M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (25m) 17s ago 25m 56.0M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (25m) 17s ago 25m 55.0M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (25m) 17s ago 25m 54.5M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (2m) 17s ago 24m 60.8M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:28:28.038 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (22m) 16s ago 22m 77.1M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:28:28.039 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (22m) 17s ago 22m 76.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:28:28.481 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:28:28.481 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:28:28.481 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:28:28.481 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:28:28.481 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:28:28.481 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:28:28.481 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:28:28.481 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:28:28.482 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T23:28:28.482 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:28:28.482 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:28:28.482 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:28:28.482 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:28:28.482 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:28:28.482 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T23:28:28.482 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 5 2024-09-13T23:28:28.482 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:28:28.482 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:28:28.737 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:28 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:28.737 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:28 smithi022 ceph-mon[80039]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:28:28.737 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:28 smithi022 ceph-mon[80039]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:28:28.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:28 smithi022 ceph-mon[80039]: from='client.54150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:28:28.738 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:28 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/2435280814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:28.738 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:28 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:28.738 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:28 smithi022 ceph-mon[82397]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:28:28.738 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:28 smithi022 ceph-mon[82397]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:28:28.738 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:28 smithi022 ceph-mon[82397]: from='client.54150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:28:28.738 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:28 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/2435280814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:28.856 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:28:28.856 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:28:28.856 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:28:28.857 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi022", 2024-09-13T23:28:28.857 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:28:28.857 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "1/2 daemons upgraded", 2024-09-13T23:28:28.857 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T23:28:28.857 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:28:28.857 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:28:29.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T23:28:29.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: mon.a calling monitor election 2024-09-13T23:28:29.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: mon.c calling monitor election 2024-09-13T23:28:29.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T23:28:29.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T23:28:29.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:28:29.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T23:28:29.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T23:28:29.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: monmap epoch 4 2024-09-13T23:28:29.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:28:29.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: last_changed 2024-09-13T23:28:21.450332+0000 2024-09-13T23:28:29.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: created 2024-09-13T22:58:24.106959+0000 2024-09-13T23:28:29.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: min_mon_release 19 (squid) 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: election_strategy: 1 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: 1: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: 2: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.b 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: fsmap 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: mgrmap e48: y(active, since 20s), standbys: x 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: overall HEALTH_OK 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:28:29.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:28:29.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: from='client.54150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:28:29.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:28 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/2435280814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: mon.b calling monitor election 2024-09-13T23:28:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: mon.a calling monitor election 2024-09-13T23:28:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: mon.c calling monitor election 2024-09-13T23:28:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T23:28:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: monmap epoch 4 2024-09-13T23:28:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:28:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: last_changed 2024-09-13T23:28:21.450332+0000 2024-09-13T23:28:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: created 2024-09-13T22:58:24.106959+0000 2024-09-13T23:28:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: min_mon_release 19 (squid) 2024-09-13T23:28:29.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: election_strategy: 1 2024-09-13T23:28:29.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-13T23:28:29.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: 1: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-13T23:28:29.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: 2: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.b 2024-09-13T23:28:29.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: fsmap 2024-09-13T23:28:29.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T23:28:29.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: mgrmap e48: y(active, since 22s), standbys: x 2024-09-13T23:28:29.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: overall HEALTH_OK 2024-09-13T23:28:29.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[80039]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:28:29.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: mon.b calling monitor election 2024-09-13T23:28:29.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: mon.a calling monitor election 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: mon.c calling monitor election 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: monmap epoch 4 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: last_changed 2024-09-13T23:28:21.450332+0000 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: created 2024-09-13T22:58:24.106959+0000 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: min_mon_release 19 (squid) 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: election_strategy: 1 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: 1: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: 2: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.b 2024-09-13T23:28:29.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: fsmap 2024-09-13T23:28:29.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T23:28:29.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: mgrmap e48: y(active, since 22s), standbys: x 2024-09-13T23:28:29.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: overall HEALTH_OK 2024-09-13T23:28:29.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:29 smithi022 ceph-mon[82397]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:28:29.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: mon.b calling monitor election 2024-09-13T23:28:29.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: mon.a calling monitor election 2024-09-13T23:28:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: mon.c calling monitor election 2024-09-13T23:28:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T23:28:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: monmap epoch 4 2024-09-13T23:28:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: fsid 9adcac8c-7223-11ef-bceb-c7b262605968 2024-09-13T23:28:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: last_changed 2024-09-13T23:28:21.450332+0000 2024-09-13T23:28:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: created 2024-09-13T22:58:24.106959+0000 2024-09-13T23:28:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: min_mon_release 19 (squid) 2024-09-13T23:28:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: election_strategy: 1 2024-09-13T23:28:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-13T23:28:29.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: 1: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-13T23:28:29.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: 2: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.b 2024-09-13T23:28:29.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: fsmap 2024-09-13T23:28:29.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T23:28:29.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: mgrmap e48: y(active, since 22s), standbys: x 2024-09-13T23:28:29.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: overall HEALTH_OK 2024-09-13T23:28:29.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:29 smithi076 ceph-mon[63913]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:28:32.452 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:32 smithi022 ceph-mon[80039]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:32.452 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:32 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:32.452 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:32 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:32.453 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:32 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:32.453 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:32 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:32.453 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:32 smithi022 ceph-mon[82397]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:32.453 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:32 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:32.453 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:32 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:32.453 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:32 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:32.453 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:32 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:32.491 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:32 smithi076 ceph-mon[63913]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:32.491 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:32 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:32.491 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:32 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:32.491 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:32 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:32.491 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:32 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:32.811 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:28:32] "GET /metrics HTTP/1.1" 200 37867 "" "Prometheus/2.43.0" 2024-09-13T23:28:34.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:34 smithi076 ceph-mon[63913]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:34.647 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:34 smithi022 ceph-mon[80039]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:34.647 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:34 smithi022 ceph-mon[82397]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[80039]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[80039]: Detected new or changed devices on smithi022 2024-09-13T23:28:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:28:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[80039]: Reconfiguring mon.a (monmap changed)... 2024-09-13T23:28:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:28:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:28:36.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:36.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[80039]: Reconfiguring daemon mon.a on smithi022 2024-09-13T23:28:36.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[82397]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:36.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[82397]: Detected new or changed devices on smithi022 2024-09-13T23:28:36.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:36.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:36.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:36.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:28:36.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:36.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[82397]: Reconfiguring mon.a (monmap changed)... 2024-09-13T23:28:36.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:28:36.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:28:36.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:36.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:36 smithi022 ceph-mon[82397]: Reconfiguring daemon mon.a on smithi022 2024-09-13T23:28:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:36 smithi076 ceph-mon[63913]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:36.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:36 smithi076 ceph-mon[63913]: Detected new or changed devices on smithi022 2024-09-13T23:28:36.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:36.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:36.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:36.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:28:36.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:36.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:36 smithi076 ceph-mon[63913]: Reconfiguring mon.a (monmap changed)... 2024-09-13T23:28:36.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:28:36.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:28:36.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:36.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:36 smithi076 ceph-mon[63913]: Reconfiguring daemon mon.a on smithi022 2024-09-13T23:28:37.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:28:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:28:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:28:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:37.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:28:37.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:37.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:37.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:28:37.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:28:37.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:28:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:37.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:37.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:28:37.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:28:37.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:38.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[80039]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:38.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[80039]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T23:28:38.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[80039]: Reconfiguring daemon mgr.y on smithi022 2024-09-13T23:28:38.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:38.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:38.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:28:38.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:28:38.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:38.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[82397]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:38.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[82397]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T23:28:38.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[82397]: Reconfiguring daemon mgr.y on smithi022 2024-09-13T23:28:38.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:38.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:38.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:28:38.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:28:38.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:38 smithi076 ceph-mon[63913]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:38 smithi076 ceph-mon[63913]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T23:28:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:38 smithi076 ceph-mon[63913]: Reconfiguring daemon mgr.y on smithi022 2024-09-13T23:28:38.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:38.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:38.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:28:38.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:28:38.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:39.242 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:39 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:39.242 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:39 smithi022 ceph-mon[82397]: Reconfiguring mon.c (monmap changed)... 2024-09-13T23:28:39.242 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:39 smithi022 ceph-mon[82397]: Reconfiguring daemon mon.c on smithi022 2024-09-13T23:28:39.242 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:39 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:39.242 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:39 smithi022 ceph-mon[80039]: Reconfiguring mon.c (monmap changed)... 2024-09-13T23:28:39.243 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:39 smithi022 ceph-mon[80039]: Reconfiguring daemon mon.c on smithi022 2024-09-13T23:28:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:39 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:39 smithi076 ceph-mon[63913]: Reconfiguring mon.c (monmap changed)... 2024-09-13T23:28:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:39 smithi076 ceph-mon[63913]: Reconfiguring daemon mon.c on smithi022 2024-09-13T23:28:40.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:40 smithi022 ceph-mon[80039]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:40.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:40 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:40.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:40 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:40.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:40 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T23:28:40.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:40 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:40.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:40 smithi022 ceph-mon[82397]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:40.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:40 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:40.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:40 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:40.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:40 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T23:28:40.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:40 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:40.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:40 smithi076 ceph-mon[63913]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:40.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:40 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:40.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:40 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:40.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:40 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T23:28:40.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:40 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:41.324 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:41 smithi022 ceph-mon[80039]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T23:28:41.324 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:41 smithi022 ceph-mon[80039]: Reconfiguring daemon osd.0 on smithi022 2024-09-13T23:28:41.324 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:41.324 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:41.325 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T23:28:41.325 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:41.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:41 smithi022 ceph-mon[82397]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T23:28:41.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:41 smithi022 ceph-mon[82397]: Reconfiguring daemon osd.0 on smithi022 2024-09-13T23:28:41.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:41.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:41.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T23:28:41.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:41 smithi076 ceph-mon[63913]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T23:28:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:41 smithi076 ceph-mon[63913]: Reconfiguring daemon osd.0 on smithi022 2024-09-13T23:28:41.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:41.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:41.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T23:28:41.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:42.355 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:42 smithi022 ceph-mon[80039]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:42.355 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:42 smithi022 ceph-mon[80039]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T23:28:42.355 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:42 smithi022 ceph-mon[80039]: Reconfiguring daemon osd.1 on smithi022 2024-09-13T23:28:42.355 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:42 smithi022 ceph-mon[82397]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:42.355 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:42 smithi022 ceph-mon[82397]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T23:28:42.356 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:42 smithi022 ceph-mon[82397]: Reconfiguring daemon osd.1 on smithi022 2024-09-13T23:28:42.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:42 smithi076 ceph-mon[63913]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:42.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:42 smithi076 ceph-mon[63913]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T23:28:42.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:42 smithi076 ceph-mon[63913]: Reconfiguring daemon osd.1 on smithi022 2024-09-13T23:28:42.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:28:42] "GET /metrics HTTP/1.1" 200 37944 "" "Prometheus/2.43.0" 2024-09-13T23:28:43.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:43 smithi076 ceph-mon[63913]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:43.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:43 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:43.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:43 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:43.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:43 smithi076 ceph-mon[63913]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T23:28:43.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:43 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T23:28:43.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:43 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:43.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:43 smithi076 ceph-mon[63913]: Reconfiguring daemon osd.2 on smithi022 2024-09-13T23:28:43.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[80039]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:43.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:43.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:43.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[80039]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T23:28:43.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T23:28:43.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:43.829 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[80039]: Reconfiguring daemon osd.2 on smithi022 2024-09-13T23:28:43.830 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[82397]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:43.830 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:43.830 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:43.830 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[82397]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T23:28:43.830 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T23:28:43.830 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:43.830 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:43 smithi022 ceph-mon[82397]: Reconfiguring daemon osd.2 on smithi022 2024-09-13T23:28:45.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:44 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:45.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:44 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:45.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:44 smithi076 ceph-mon[63913]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T23:28:45.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:44 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T23:28:45.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:44 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:45.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:44 smithi076 ceph-mon[63913]: Reconfiguring daemon osd.3 on smithi022 2024-09-13T23:28:45.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:44 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:45.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:44 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:45.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:44 smithi022 ceph-mon[80039]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T23:28:45.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:44 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T23:28:45.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:44 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:45.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:44 smithi022 ceph-mon[80039]: Reconfiguring daemon osd.3 on smithi022 2024-09-13T23:28:45.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:44 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:45.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:44 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:45.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:44 smithi022 ceph-mon[82397]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T23:28:45.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:44 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T23:28:45.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:44 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:45.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:44 smithi022 ceph-mon[82397]: Reconfiguring daemon osd.3 on smithi022 2024-09-13T23:28:46.063 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:45 smithi022 ceph-mon[80039]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:46.063 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:46.063 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:46.063 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi022.snmrkm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:28:46.063 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:46.063 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:45 smithi022 ceph-mon[82397]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:46.063 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:46.064 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:46.064 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi022.snmrkm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:28:46.064 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:45 smithi076 ceph-mon[63913]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:46.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi022.snmrkm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:28:46.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:47.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[80039]: Reconfiguring rgw.foo.smithi022.snmrkm (monmap changed)... 2024-09-13T23:28:47.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[80039]: Reconfiguring daemon rgw.foo.smithi022.snmrkm on smithi022 2024-09-13T23:28:47.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:47.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:47.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:28:47.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:28:47.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:47.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[82397]: Reconfiguring rgw.foo.smithi022.snmrkm (monmap changed)... 2024-09-13T23:28:47.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[82397]: Reconfiguring daemon rgw.foo.smithi022.snmrkm on smithi022 2024-09-13T23:28:47.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:47.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:47.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:28:47.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:28:47.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:46 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:46 smithi076 ceph-mon[63913]: Reconfiguring rgw.foo.smithi022.snmrkm (monmap changed)... 2024-09-13T23:28:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:46 smithi076 ceph-mon[63913]: Reconfiguring daemon rgw.foo.smithi022.snmrkm on smithi022 2024-09-13T23:28:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:46 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:46 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:47.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:46 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T23:28:47.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:46 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T23:28:47.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:46 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:48.196 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:47 smithi076 ceph-mon[63913]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:48.197 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:47 smithi076 ceph-mon[63913]: Reconfiguring mon.b (monmap changed)... 2024-09-13T23:28:48.197 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:47 smithi076 ceph-mon[63913]: Reconfiguring daemon mon.b on smithi076 2024-09-13T23:28:48.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:47 smithi022 ceph-mon[80039]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:48.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:47 smithi022 ceph-mon[80039]: Reconfiguring mon.b (monmap changed)... 2024-09-13T23:28:48.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:47 smithi022 ceph-mon[80039]: Reconfiguring daemon mon.b on smithi076 2024-09-13T23:28:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:47 smithi022 ceph-mon[82397]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:47 smithi022 ceph-mon[82397]: Reconfiguring mon.b (monmap changed)... 2024-09-13T23:28:48.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:47 smithi022 ceph-mon[82397]: Reconfiguring daemon mon.b on smithi076 2024-09-13T23:28:49.229 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:48 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:49.229 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:48 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:49.229 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:48 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:49.229 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:48 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:28:49.229 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:48 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:28:49.229 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:48 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:49.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:48 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:49.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:48 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:49.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:48 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:49.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:48 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:28:49.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:48 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:28:49.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:48 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:49.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:48 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:49.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:48 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:49.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:48 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:49.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:48 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T23:28:49.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:48 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T23:28:49.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:48 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:49 smithi076 ceph-mon[63913]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:50.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:49 smithi076 ceph-mon[63913]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T23:28:50.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:49 smithi076 ceph-mon[63913]: Reconfiguring daemon mgr.x on smithi076 2024-09-13T23:28:50.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:50.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:50.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T23:28:50.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:50.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[80039]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:50.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[80039]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T23:28:50.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[80039]: Reconfiguring daemon mgr.x on smithi076 2024-09-13T23:28:50.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:50.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:50.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T23:28:50.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:50.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[82397]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:50.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[82397]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T23:28:50.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[82397]: Reconfiguring daemon mgr.x on smithi076 2024-09-13T23:28:50.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:50.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:50.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T23:28:50.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:51.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:50 smithi076 ceph-mon[63913]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T23:28:51.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:50 smithi076 ceph-mon[63913]: Reconfiguring daemon osd.4 on smithi076 2024-09-13T23:28:51.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:50 smithi022 ceph-mon[80039]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T23:28:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:50 smithi022 ceph-mon[80039]: Reconfiguring daemon osd.4 on smithi076 2024-09-13T23:28:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:50 smithi022 ceph-mon[82397]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T23:28:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:50 smithi022 ceph-mon[82397]: Reconfiguring daemon osd.4 on smithi076 2024-09-13T23:28:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:52 smithi076 ceph-mon[63913]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:52 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:52.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:52 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:52.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:52 smithi076 ceph-mon[63913]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T23:28:52.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:52 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T23:28:52.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:52 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:52.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:52 smithi076 ceph-mon[63913]: Reconfiguring daemon osd.5 on smithi076 2024-09-13T23:28:52.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:52 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:28:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[80039]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[80039]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T23:28:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T23:28:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[80039]: Reconfiguring daemon osd.5 on smithi076 2024-09-13T23:28:52.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:28:52.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[82397]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:52.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:52.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:52.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[82397]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T23:28:52.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T23:28:52.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:52.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[82397]: Reconfiguring daemon osd.5 on smithi076 2024-09-13T23:28:52.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:28:52.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:28:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:28:52] "GET /metrics HTTP/1.1" 200 37946 "" "Prometheus/2.43.0" 2024-09-13T23:28:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[80039]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[80039]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T23:28:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T23:28:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[80039]: Reconfiguring daemon osd.6 on smithi076 2024-09-13T23:28:53.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[82397]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:53.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:53.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:53.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[82397]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T23:28:53.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T23:28:53.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:53.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:53 smithi022 ceph-mon[82397]: Reconfiguring daemon osd.6 on smithi076 2024-09-13T23:28:54.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:53 smithi076 ceph-mon[63913]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:28:54.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:53 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:54.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:53 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:54.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:53 smithi076 ceph-mon[63913]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T23:28:54.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:53 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T23:28:54.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:53 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:54.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:53 smithi076 ceph-mon[63913]: Reconfiguring daemon osd.6 on smithi076 2024-09-13T23:28:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:55 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:55 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:55 smithi022 ceph-mon[82397]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T23:28:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:55 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T23:28:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:55 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:55 smithi022 ceph-mon[82397]: Reconfiguring daemon osd.7 on smithi076 2024-09-13T23:28:55.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:55 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:55.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:55 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:55.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:55 smithi022 ceph-mon[80039]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T23:28:55.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:55 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T23:28:55.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:55 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:55.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:55 smithi022 ceph-mon[80039]: Reconfiguring daemon osd.7 on smithi076 2024-09-13T23:28:55.488 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:55 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:55.488 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:55 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:55.488 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:55 smithi076 ceph-mon[63913]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T23:28:55.488 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:55 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T23:28:55.488 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:55 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:55.488 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:55 smithi076 ceph-mon[63913]: Reconfiguring daemon osd.7 on smithi076 2024-09-13T23:28:56.208 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:56 smithi076 ceph-mon[63913]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:56.209 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:56 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:56.209 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:56 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:56.209 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:56 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi076.rnamrn", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:28:56.209 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:56 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:56.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:56 smithi022 ceph-mon[80039]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:56.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:56 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:56.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:56 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:56.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:56 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi076.rnamrn", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:28:56.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:56 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:56 smithi022 ceph-mon[82397]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:56 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:56 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:56.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:56 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi076.rnamrn", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:28:56.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:56 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: Reconfiguring rgw.foo.smithi076.rnamrn (monmap changed)... 2024-09-13T23:28:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: Reconfiguring daemon rgw.foo.smithi076.rnamrn on smithi076 2024-09-13T23:28:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:28:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T23:28:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T23:28:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: Reconfiguring rgw.foo.smithi076.rnamrn (monmap changed)... 2024-09-13T23:28:57.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: Reconfiguring daemon rgw.foo.smithi076.rnamrn on smithi076 2024-09-13T23:28:57.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:28:57.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T23:28:57.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T23:28:57.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.434 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.434 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.434 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.434 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.434 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:28:57.434 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:28:57.434 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.434 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:28:57.434 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:28:57.434 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:28:57.434 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:28:57.434 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:28:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:28:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:28:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:28:57.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:28:57.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:28:57.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.439 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:28:57.439 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:28:57.439 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:28:57.439 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:28:57.439 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.439 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:28:57.439 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:28:57.439 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:28:57.439 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:28:57.439 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:28:57.439 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:28:57.440 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:28:57.440 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.440 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.440 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.440 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.440 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.440 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.440 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:28:57.440 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:28:57.440 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:57.440 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:28:57.440 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: Reconfiguring rgw.foo.smithi076.rnamrn (monmap changed)... 2024-09-13T23:28:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: Reconfiguring daemon rgw.foo.smithi076.rnamrn on smithi076 2024-09-13T23:28:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:28:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.540 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:28:57.541 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:28:57.542 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:28:57.543 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:57.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:28:57.544 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mon 2024-09-13T23:28:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all crash 2024-09-13T23:28:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mds 2024-09-13T23:28:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:28:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T23:28:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:28:58.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all nfs 2024-09-13T23:28:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:28:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all node-exporter 2024-09-13T23:28:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all prometheus 2024-09-13T23:28:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all alertmanager 2024-09-13T23:28:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all grafana 2024-09-13T23:28:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all loki 2024-09-13T23:28:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all promtail 2024-09-13T23:28:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Finalizing container_image settings 2024-09-13T23:28:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: Upgrade: Complete! 2024-09-13T23:28:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:28:58.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mon 2024-09-13T23:28:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all crash 2024-09-13T23:28:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mds 2024-09-13T23:28:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:28:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T23:28:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:28:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all nfs 2024-09-13T23:28:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:28:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all node-exporter 2024-09-13T23:28:58.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all prometheus 2024-09-13T23:28:58.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all alertmanager 2024-09-13T23:28:58.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all grafana 2024-09-13T23:28:58.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all loki 2024-09-13T23:28:58.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all promtail 2024-09-13T23:28:58.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Finalizing container_image settings 2024-09-13T23:28:58.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: Upgrade: Complete! 2024-09-13T23:28:58.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:58.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:28:58.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:58 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:58.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:28:58.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mon 2024-09-13T23:28:58.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all crash 2024-09-13T23:28:58.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mds 2024-09-13T23:28:58.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:28:58.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T23:28:58.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:28:58.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all nfs 2024-09-13T23:28:58.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:28:58.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all node-exporter 2024-09-13T23:28:58.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all prometheus 2024-09-13T23:28:58.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all alertmanager 2024-09-13T23:28:58.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all grafana 2024-09-13T23:28:58.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all loki 2024-09-13T23:28:58.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all promtail 2024-09-13T23:28:58.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Finalizing container_image settings 2024-09-13T23:28:58.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: Upgrade: Complete! 2024-09-13T23:28:58.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:28:58.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:28:58.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:58 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:28:59.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:28:59 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:59.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:28:59 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:59.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:28:59 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:28:59.880 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:29:00.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:00 smithi022 ceph-mon[80039]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:00.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:00 smithi022 ceph-mon[80039]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:00.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:00 smithi022 ceph-mon[82397]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:00.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:00 smithi022 ceph-mon[82397]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:00 smithi076 ceph-mon[63913]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:00 smithi076 ceph-mon[63913]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:01.911 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:29:01.911 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (16m) 30s ago 24m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:29:01.911 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (2m) 50s ago 23m 82.8M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:29:01.911 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (3m) 30s ago 23m 54.4M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:29:01.911 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (3m) 50s ago 28m 437M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:29:01.911 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (12m) 30s ago 30m 488M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:29:01.911 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (62s) 30s ago 30m 40.2M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:29:01.911 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (114s) 50s ago 29m 35.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:29:01.911 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (41s) 30s ago 29m 21.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:29:01.911 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (16m) 30s ago 24m 16.0M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:29:01.912 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (15m) 50s ago 24m 16.1M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:29:01.912 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (28m) 30s ago 28m 52.9M 2993M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:29:01.912 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (28m) 30s ago 28m 55.5M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:29:01.912 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (27m) 30s ago 27m 51.0M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:29:01.912 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (27m) 30s ago 27m 53.2M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:29:01.912 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (26m) 50s ago 26m 56.1M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:29:01.912 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (26m) 50s ago 26m 56.0M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:29:01.912 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (26m) 50s ago 26m 55.0M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:29:01.912 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (25m) 50s ago 25m 54.5M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:29:01.912 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (3m) 50s ago 24m 60.8M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:29:01.912 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (23m) 30s ago 23m 77.5M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:29:01.912 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (23m) 50s ago 23m 76.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:29:02.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:02 smithi022 ceph-mon[80039]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:02.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:02 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:02.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:02 smithi022 ceph-mon[82397]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:02.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:02 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:02.508 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-09-13T23:29:02.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:02 smithi076 ceph-mon[63913]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:02.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:02 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:02.765 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:29:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:29:02] "GET /metrics HTTP/1.1" 200 37946 "" "Prometheus/2.43.0" 2024-09-13T23:29:03.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:03 smithi022 ceph-mon[80039]: from='client.64108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:03.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:03 smithi022 ceph-mon[82397]: from='client.64108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:03.519 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:03 smithi076 ceph-mon[63913]: from='client.64108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:04.275 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:04 smithi022 ceph-mon[80039]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:04.275 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:04 smithi022 ceph-mon[82397]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:04 smithi076 ceph-mon[63913]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:04.700 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:29:05.288 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:05 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/5778856' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:05.288 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:05 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/5778856' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:05.314 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-09-13T23:29:05.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:05 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/5778856' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:06.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:06 smithi076 ceph-mon[63913]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:06.553 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:06 smithi022 ceph-mon[80039]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:06.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:06 smithi022 ceph-mon[82397]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:07.346 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)" 2024-09-13T23:29:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:29:07.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:29:07.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:29:07.995 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-09-13T23:29:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:08 smithi076 ceph-mon[63913]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:08 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/44100634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:08.538 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:08 smithi022 ceph-mon[80039]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:08.538 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:08 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/44100634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:08.538 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:08 smithi022 ceph-mon[82397]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:08.538 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:08 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/44100634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:09.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:09 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:09.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:09 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:09 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:10.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:10 smithi022 ceph-mon[80039]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:10.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:10 smithi022 ceph-mon[82397]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:10.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:10 smithi076 ceph-mon[63913]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:11.376 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:11 smithi022 ceph-mon[80039]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:11.377 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:11 smithi022 ceph-mon[82397]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:11.423 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:29:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:11 smithi076 ceph-mon[63913]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:12.051 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T23:29:12.364 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:12 smithi022 ceph-mon[80039]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:12.364 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:12 smithi022 ceph-mon[82397]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:12 smithi076 ceph-mon[63913]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:12.675 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:29:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:29:12] "GET /metrics HTTP/1.1" 200 37946 "" "Prometheus/2.43.0" 2024-09-13T23:29:14.045 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:29:14.045 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": null, 2024-09-13T23:29:14.046 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": false, 2024-09-13T23:29:14.046 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "", 2024-09-13T23:29:14.046 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:29:14.046 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": null, 2024-09-13T23:29:14.046 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-09-13T23:29:14.046 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:29:14.046 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:29:14.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:14 smithi022 ceph-mon[80039]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:14 smithi022 ceph-mon[82397]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:14.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:14 smithi076 ceph-mon[63913]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:14.619 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T23:29:15.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:15 smithi076 ceph-mon[63913]: from='client.54198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:15.602 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:15 smithi022 ceph-mon[82397]: from='client.54198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:15.602 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:15 smithi022 ceph-mon[80039]: from='client.54198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:16.249 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:16 smithi022 ceph-mon[80039]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:16.249 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:16 smithi022 ceph-mon[82397]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:16 smithi076 ceph-mon[63913]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:16.661 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-09-13T23:29:17.211 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-09-13T23:29:17.522 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:17 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/3726731868' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:29:17.522 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:17 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/3726731868' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:29:17.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:17 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/3726731868' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:29:18.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:18 smithi076 ceph-mon[63913]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:18.556 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:18 smithi022 ceph-mon[80039]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:18.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:18 smithi022 ceph-mon[82397]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:19 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:19.637 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:19 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:19.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:19 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:20.537 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:20 smithi022 ceph-mon[80039]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:20.537 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:20 smithi022 ceph-mon[82397]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:20.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:20 smithi076 ceph-mon[63913]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:20.571 INFO:teuthology.orchestra.run.smithi022.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:29:21.266 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[80039]: from='client.64144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:21.267 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:21.267 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:21.267 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:21.267 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:29:21.267 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:29:21.267 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:21.267 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[82397]: from='client.64144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:21.268 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:21.268 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:21.268 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:21.268 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:29:21.268 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:29:21.268 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:21.293 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T23:29:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:21 smithi076 ceph-mon[63913]: from='client.64144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:21.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:29:21.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:29:21.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:22.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:22 smithi022 ceph-mon[80039]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:22.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:22 smithi022 ceph-mon[80039]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:29:22.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:22 smithi022 ceph-mon[80039]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:29:22.382 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:29:22.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:22 smithi022 ceph-mon[82397]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:22.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:22 smithi022 ceph-mon[82397]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:29:22.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:22 smithi022 ceph-mon[82397]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:29:22.383 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:29:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:22 smithi076 ceph-mon[63913]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:22 smithi076 ceph-mon[63913]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:29:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:22 smithi076 ceph-mon[63913]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:29:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:29:22.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:29:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:29:22] "GET /metrics HTTP/1.1" 200 37944 "" "Prometheus/2.43.0" 2024-09-13T23:29:23.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:23.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:23.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:29:23.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:29:23.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:29:23.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mgr 2024-09-13T23:29:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mon 2024-09-13T23:29:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all crash 2024-09-13T23:29:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T23:29:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T23:29:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[82397]: Upgrade: osd.0 is safe to restart 2024-09-13T23:29:23.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:23.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:23.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:29:23.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:29:23.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:29:23.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:23.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mgr 2024-09-13T23:29:23.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:23.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:23.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mon 2024-09-13T23:29:23.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:23.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:23.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all crash 2024-09-13T23:29:23.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:23.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T23:29:23.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T23:29:23.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:23 smithi022 ceph-mon[80039]: Upgrade: osd.0 is safe to restart 2024-09-13T23:29:23.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:23.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:23.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:29:23.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:29:23.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:29:23.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:23.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mgr 2024-09-13T23:29:23.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:23.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:23.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mon 2024-09-13T23:29:23.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:23.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:23.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all crash 2024-09-13T23:29:23.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:23.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T23:29:23.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T23:29:23.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:23 smithi076 ceph-mon[63913]: Upgrade: osd.0 is safe to restart 2024-09-13T23:29:24.523 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:29:25.221 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:29:25.221 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (16m) 54s ago 24m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:29:25.221 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (3m) 74s ago 24m 82.8M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:29:25.221 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (3m) 54s ago 23m 54.4M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:29:25.221 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (3m) 74s ago 29m 437M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:29:25.221 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (13m) 54s ago 30m 488M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:29:25.221 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (85s) 54s ago 30m 40.2M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:29:25.221 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (2m) 74s ago 29m 35.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:29:25.221 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (64s) 54s ago 29m 21.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (16m) 54s ago 25m 16.0M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (16m) 74s ago 25m 16.1M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (28m) 54s ago 28m 52.9M 2993M 17.2.0 e1d6a67b021e 8c75f288153f 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (28m) 54s ago 28m 55.5M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (28m) 54s ago 28m 51.0M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (27m) 54s ago 27m 53.2M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (27m) 74s ago 27m 56.1M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (26m) 74s ago 26m 56.0M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (26m) 74s ago 26m 55.0M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (26m) 74s ago 26m 54.5M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (3m) 74s ago 25m 60.8M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (23m) 54s ago 23m 77.5M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:29:25.222 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (23m) 74s ago 23m 76.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:29:25.653 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:29:25.654 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:29:25.654 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:29:25.654 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:29:25.654 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:29:25.654 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:29:25.654 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:29:25.654 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:29:25.654 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T23:29:25.654 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:29:25.654 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:29:25.654 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:29:25.654 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:29:25.655 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:29:25.655 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T23:29:25.655 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 5 2024-09-13T23:29:25.655 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:29:25.655 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:29:26.013 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:29:26.013 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:29:26.013 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:29:26.013 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-09-13T23:29:26.013 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:29:26.013 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "0/8 daemons upgraded", 2024-09-13T23:29:26.013 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T23:29:26.013 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:29:26.013 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:29:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[80039]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[80039]: from='client.54216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[80039]: from='client.64156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[80039]: Upgrade: Updating osd.0 2024-09-13T23:29:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T23:29:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:29:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[80039]: Deploying daemon osd.0 on smithi022 2024-09-13T23:29:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[80039]: from='client.64162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/3836693327' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:26.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[82397]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:26.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[82397]: from='client.54216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:26.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[82397]: from='client.64156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:26.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[82397]: Upgrade: Updating osd.0 2024-09-13T23:29:26.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:26.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T23:29:26.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:29:26.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[82397]: Deploying daemon osd.0 on smithi022 2024-09-13T23:29:26.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[82397]: from='client.64162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:26.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:26 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/3836693327' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:26 smithi076 ceph-mon[63913]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:26 smithi076 ceph-mon[63913]: from='client.54216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:26 smithi076 ceph-mon[63913]: from='client.64156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:26 smithi076 ceph-mon[63913]: Upgrade: Updating osd.0 2024-09-13T23:29:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:26 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:26 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T23:29:26.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:26 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:29:26.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:26 smithi076 ceph-mon[63913]: Deploying daemon osd.0 on smithi022 2024-09-13T23:29:26.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:26 smithi076 ceph-mon[63913]: from='client.64162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:26.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:26 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/3836693327' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:27.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:27 smithi022 ceph-mon[82397]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:27.321 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:27 smithi022 ceph-mon[80039]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:27 smithi076 ceph-mon[63913]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:28.279 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:28 smithi022 ceph-mon[80039]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:28.279 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:28 smithi022 ceph-mon[82397]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:28.280 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:28 smithi022 systemd[1]: Stopping Ceph osd.0 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:29:28.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:28 smithi076 ceph-mon[63913]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:28.676 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:28 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0[31271]: 2024-09-13T23:29:28.277+0000 7f1f6b6ce700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:29:28.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:28 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0[31271]: 2024-09-13T23:29:28.277+0000 7f1f6b6ce700 -1 osd.0 113 *** Got signal Terminated *** 2024-09-13T23:29:28.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:28 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0[31271]: 2024-09-13T23:29:28.277+0000 7f1f6b6ce700 -1 osd.0 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:29:29.405 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:29 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:29.406 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:29 smithi022 ceph-mon[80039]: osd.0 marked itself down and dead 2024-09-13T23:29:29.406 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:29 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:29.406 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:29 smithi022 ceph-mon[82397]: osd.0 marked itself down and dead 2024-09-13T23:29:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:29 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:29 smithi076 ceph-mon[63913]: osd.0 marked itself down and dead 2024-09-13T23:29:29.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:29 smithi022 podman[89254]: 2024-09-13 23:29:29.40483399 +0000 UTC m=+1.234550986 container died 8c75f288153fb35766fbaa8d0785b6a946c1d3457a05463bda80b46ed942b90e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, ceph=True, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., name=centos-stream, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, RELEASE=HEAD, version=8, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-13T23:29:30.032 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:29 smithi022 podman[89254]: 2024-09-13 23:29:29.827673507 +0000 UTC m=+1.657390297 container cleanup 8c75f288153fb35766fbaa8d0785b6a946c1d3457a05463bda80b46ed942b90e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , release=754, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, version=8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream) 2024-09-13T23:29:30.033 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:29 smithi022 bash[89254]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0 2024-09-13T23:29:30.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:30 smithi022 ceph-mon[80039]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:30.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:30 smithi022 ceph-mon[80039]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:29:30.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:30 smithi022 ceph-mon[80039]: osdmap e114: 8 total, 7 up, 8 in 2024-09-13T23:29:30.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:30 smithi022 ceph-mon[82397]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:30.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:30 smithi022 ceph-mon[82397]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:29:30.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:30 smithi022 ceph-mon[82397]: osdmap e114: 8 total, 7 up, 8 in 2024-09-13T23:29:30.428 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:30 smithi022 podman[89274]: 2024-09-13 23:29:30.028925865 +0000 UTC m=+0.616432886 container remove 8c75f288153fb35766fbaa8d0785b6a946c1d3457a05463bda80b46ed942b90e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, version=8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T23:29:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:30 smithi076 ceph-mon[63913]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:30 smithi076 ceph-mon[63913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:29:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:30 smithi076 ceph-mon[63913]: osdmap e114: 8 total, 7 up, 8 in 2024-09-13T23:29:30.750 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:30 smithi022 podman[89362]: 2024-09-13 23:29:30.374881437 +0000 UTC m=+0.016417374 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:29:30.750 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:30 smithi022 podman[89362]: 2024-09-13 23:29:30.4971327 +0000 UTC m=+0.138668671 container create 0308a9a840f314d4ef9c4321337e12fba93ead1826b3a9cff22fb301ef1f1125 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-13T23:29:30.750 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:30 smithi022 podman[89362]: 2024-09-13 23:29:30.749878315 +0000 UTC m=+0.391414251 container init 0308a9a840f314d4ef9c4321337e12fba93ead1826b3a9cff22fb301ef1f1125 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:29:31.088 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:30 smithi022 podman[89362]: 2024-09-13 23:29:30.753579131 +0000 UTC m=+0.395115058 container start 0308a9a840f314d4ef9c4321337e12fba93ead1826b3a9cff22fb301ef1f1125 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be) 2024-09-13T23:29:31.088 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:30 smithi022 podman[89362]: 2024-09-13 23:29:30.874720815 +0000 UTC m=+0.516256758 container attach 0308a9a840f314d4ef9c4321337e12fba93ead1826b3a9cff22fb301ef1f1125 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-13T23:29:31.088 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:30 smithi022 podman[89362]: 2024-09-13 23:29:30.968558394 +0000 UTC m=+0.610094326 container died 0308a9a840f314d4ef9c4321337e12fba93ead1826b3a9cff22fb301ef1f1125 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, ceph=True, GIT_CLEAN=True) 2024-09-13T23:29:31.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:31 smithi022 ceph-mon[80039]: osdmap e115: 8 total, 7 up, 8 in 2024-09-13T23:29:31.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:31 smithi022 ceph-mon[82397]: osdmap e115: 8 total, 7 up, 8 in 2024-09-13T23:29:31.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:31 smithi076 ceph-mon[63913]: osdmap e115: 8 total, 7 up, 8 in 2024-09-13T23:29:31.926 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:31 smithi022 podman[89384]: 2024-09-13 23:29:31.653775946 +0000 UTC m=+0.679357211 container remove 0308a9a840f314d4ef9c4321337e12fba93ead1826b3a9cff22fb301ef1f1125 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-13T23:29:31.927 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:31 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T23:29:31.927 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:31 smithi022 systemd[1]: Stopped Ceph osd.0 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:29:31.927 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:31 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.0.service: Consumed 14.625s CPU time. 2024-09-13T23:29:31.927 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:31 smithi022 systemd[1]: Starting Ceph osd.0 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:29:32.366 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:32 smithi022 ceph-mon[80039]: pgmap v47: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-13T23:29:32.367 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:32 smithi022 ceph-mon[82397]: pgmap v47: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-13T23:29:32.367 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:32 smithi022 podman[89485]: 2024-09-13 23:29:32.016701041 +0000 UTC m=+0.016525059 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:29:32.367 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:32 smithi022 podman[89485]: 2024-09-13 23:29:32.130154572 +0000 UTC m=+0.129978638 container create 91cdd29a9483633147b099774aa324bf55f76e640005671ede8d097a2f30b182 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be) 2024-09-13T23:29:32.367 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:32 smithi022 podman[89485]: 2024-09-13 23:29:32.366209136 +0000 UTC m=+0.366033156 container init 91cdd29a9483633147b099774aa324bf55f76e640005671ede8d097a2f30b182 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-13T23:29:32.439 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:32 smithi076 ceph-mon[63913]: pgmap v47: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-13T23:29:32.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:29:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:29:32] "GET /metrics HTTP/1.1" 200 37944 "" "Prometheus/2.43.0" 2024-09-13T23:29:32.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:32 smithi022 podman[89485]: 2024-09-13 23:29:32.370309109 +0000 UTC m=+0.370133131 container start 91cdd29a9483633147b099774aa324bf55f76e640005671ede8d097a2f30b182 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:29:32.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:32 smithi022 podman[89485]: 2024-09-13 23:29:32.474814775 +0000 UTC m=+0.474638809 container attach 91cdd29a9483633147b099774aa324bf55f76e640005671ede8d097a2f30b182 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:29:32.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate[89500]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:29:32.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:32 smithi022 bash[89485]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:29:32.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate[89500]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:29:32.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:32 smithi022 bash[89485]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:29:33.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:33 smithi022 ceph-mon[80039]: Health check failed: Degraded data redundancy: 67/627 objects degraded (10.686%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T23:29:33.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:33 smithi022 ceph-mon[82397]: Health check failed: Degraded data redundancy: 67/627 objects degraded (10.686%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T23:29:33.455 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:33 smithi076 ceph-mon[63913]: Health check failed: Degraded data redundancy: 67/627 objects degraded (10.686%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T23:29:34.142 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate[89500]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T23:29:34.142 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:33 smithi022 bash[89485]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T23:29:34.142 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:33 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate[89500]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T23:29:34.143 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:33 smithi022 bash[89485]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T23:29:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:34 smithi076 ceph-mon[63913]: pgmap v48: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-09-13T23:29:34.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:34 smithi022 ceph-mon[80039]: pgmap v48: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-09-13T23:29:34.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:34 smithi022 ceph-mon[82397]: pgmap v48: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-09-13T23:29:34.427 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate[89500]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T23:29:34.427 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:34 smithi022 bash[89485]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T23:29:34.427 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate[89500]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T23:29:34.427 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:34 smithi022 bash[89485]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T23:29:34.427 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate[89500]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T23:29:34.427 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:34 smithi022 bash[89485]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T23:29:34.428 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate[89500]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T23:29:34.428 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:34 smithi022 bash[89485]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T23:29:34.428 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:34 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate[89500]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T23:29:34.428 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:34 smithi022 bash[89485]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T23:29:34.428 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:34 smithi022 podman[89485]: 2024-09-13 23:29:34.211620541 +0000 UTC m=+2.211444572 container died 91cdd29a9483633147b099774aa324bf55f76e640005671ede8d097a2f30b182 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:29:35.129 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:34 smithi022 podman[89643]: 2024-09-13 23:29:34.812481685 +0000 UTC m=+0.594673324 container remove 91cdd29a9483633147b099774aa324bf55f76e640005671ede8d097a2f30b182 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T23:29:35.129 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:35 smithi022 podman[89704]: 2024-09-13 23:29:34.924344213 +0000 UTC m=+0.016572511 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:29:35.129 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:35 smithi022 podman[89704]: 2024-09-13 23:29:35.030306705 +0000 UTC m=+0.122534992 container create ad68a450511362fd57ebf9de49531cfc3d3c592d023331a0074ec2a2751d0445 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:29:35.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:35 smithi022 podman[89704]: 2024-09-13 23:29:35.250130155 +0000 UTC m=+0.342358454 container init ad68a450511362fd57ebf9de49531cfc3d3c592d023331a0074ec2a2751d0445 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-13T23:29:35.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:35 smithi022 podman[89704]: 2024-09-13 23:29:35.253690622 +0000 UTC m=+0.345918911 container start ad68a450511362fd57ebf9de49531cfc3d3c592d023331a0074ec2a2751d0445 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:29:35.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:35 smithi022 ceph-osd[89723]: -- 172.21.15.22:0/3129908655 <== mon.0 v2:172.21.15.22:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b52b287a40 con 0x55b52b265c00 2024-09-13T23:29:35.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:35 smithi022 bash[89704]: ad68a450511362fd57ebf9de49531cfc3d3c592d023331a0074ec2a2751d0445 2024-09-13T23:29:35.677 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:35 smithi022 systemd[1]: Started Ceph osd.0 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:29:36.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:36 smithi022 ceph-mon[80039]: pgmap v49: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-09-13T23:29:36.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:36.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:36.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:36.422 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:36 smithi022 ceph-mon[82397]: pgmap v49: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-09-13T23:29:36.422 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:36.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:36.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:36 smithi076 ceph-mon[63913]: pgmap v49: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-09-13T23:29:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:36.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:36.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:36.676 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:36 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0[89719]: 2024-09-13T23:29:36.418+0000 7f0380b57640 -1 Falling back to public interface 2024-09-13T23:29:37.660 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:37 smithi022 ceph-mon[80039]: pgmap v50: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-09-13T23:29:37.660 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:37.660 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:29:37.661 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:37 smithi022 ceph-mon[82397]: pgmap v50: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-09-13T23:29:37.661 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:37.661 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:29:37.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:37 smithi076 ceph-mon[63913]: pgmap v50: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-09-13T23:29:37.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:37.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:29:38.640 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:38 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:38.640 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:38 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:38.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:38 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:39.352 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:39 smithi022 ceph-mon[80039]: pgmap v51: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 446 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-09-13T23:29:39.352 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:39 smithi022 ceph-mon[82397]: pgmap v51: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 446 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-09-13T23:29:39.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:39 smithi076 ceph-mon[63913]: pgmap v51: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 446 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-09-13T23:29:40.176 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:39 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0[89719]: 2024-09-13T23:29:39.723+0000 7f0380b57640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-13T23:29:40.177 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:39 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0[89719]: 2024-09-13T23:29:39.746+0000 7f0380b57640 -1 osd.0 113 log_to_monitors true 2024-09-13T23:29:40.609 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:40 smithi022 ceph-mon[80039]: from='osd.0 [v2:172.21.15.22:6802/3082674401,v1:172.21.15.22:6803/3082674401]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T23:29:40.609 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:40 smithi022 ceph-mon[80039]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T23:29:40.609 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:40 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:40.609 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:40 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:40.609 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:40 smithi022 ceph-mon[82397]: from='osd.0 [v2:172.21.15.22:6802/3082674401,v1:172.21.15.22:6803/3082674401]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T23:29:40.609 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:40 smithi022 ceph-mon[82397]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T23:29:40.609 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:40 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:40.609 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:40 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:40.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:40 smithi076 ceph-mon[63913]: from='osd.0 [v2:172.21.15.22:6802/3082674401,v1:172.21.15.22:6803/3082674401]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T23:29:40.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:40 smithi076 ceph-mon[63913]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T23:29:40.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:40 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:40.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:40 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:40.926 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:29:40 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0[89719]: 2024-09-13T23:29:40.607+0000 7f03748fa640 -1 osd.0 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:29:41.636 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[80039]: pgmap v52: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 402 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-09-13T23:29:41.636 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[80039]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T23:29:41.636 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[80039]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T23:29:41.636 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[80039]: from='osd.0 [v2:172.21.15.22:6802/3082674401,v1:172.21.15.22:6803/3082674401]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:29:41.636 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[80039]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:29:41.636 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[80039]: from='osd.0 ' entity='osd.0' 2024-09-13T23:29:41.636 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:41.636 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:41.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[82397]: pgmap v52: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 402 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-09-13T23:29:41.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[82397]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T23:29:41.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[82397]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T23:29:41.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[82397]: from='osd.0 [v2:172.21.15.22:6802/3082674401,v1:172.21.15.22:6803/3082674401]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:29:41.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[82397]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:29:41.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[82397]: from='osd.0 ' entity='osd.0' 2024-09-13T23:29:41.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:41.637 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:41.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:41 smithi076 ceph-mon[63913]: pgmap v52: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 402 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-09-13T23:29:41.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:41 smithi076 ceph-mon[63913]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T23:29:41.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:41 smithi076 ceph-mon[63913]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T23:29:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:41 smithi076 ceph-mon[63913]: from='osd.0 [v2:172.21.15.22:6802/3082674401,v1:172.21.15.22:6803/3082674401]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:29:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:41 smithi076 ceph-mon[63913]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:29:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:41 smithi076 ceph-mon[63913]: from='osd.0 ' entity='osd.0' 2024-09-13T23:29:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:42.667 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:29:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:29:42] "GET /metrics HTTP/1.1" 200 38077 "" "Prometheus/2.43.0" 2024-09-13T23:29:42.667 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:42 smithi022 ceph-mon[80039]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:29:42.667 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:42 smithi022 ceph-mon[80039]: osd.0 [v2:172.21.15.22:6802/3082674401,v1:172.21.15.22:6803/3082674401] boot 2024-09-13T23:29:42.667 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:42 smithi022 ceph-mon[80039]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T23:29:42.668 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:29:42.668 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:42 smithi022 ceph-mon[82397]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:29:42.668 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:42 smithi022 ceph-mon[82397]: osd.0 [v2:172.21.15.22:6802/3082674401,v1:172.21.15.22:6803/3082674401] boot 2024-09-13T23:29:42.668 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:42 smithi022 ceph-mon[82397]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T23:29:42.668 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:29:42.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:42 smithi076 ceph-mon[63913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:29:42.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:42 smithi076 ceph-mon[63913]: osd.0 [v2:172.21.15.22:6802/3082674401,v1:172.21.15.22:6803/3082674401] boot 2024-09-13T23:29:42.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:42 smithi076 ceph-mon[63913]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T23:29:42.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T23:29:43.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:43 smithi022 ceph-mon[80039]: pgmap v55: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T23:29:43.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:43 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T23:29:43.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:43 smithi022 ceph-mon[80039]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T23:29:43.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:43 smithi022 ceph-mon[82397]: pgmap v55: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T23:29:43.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:43 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T23:29:43.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:43 smithi022 ceph-mon[82397]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T23:29:43.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:43 smithi076 ceph-mon[63913]: pgmap v55: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T23:29:43.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:43 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T23:29:43.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:43 smithi076 ceph-mon[63913]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T23:29:44.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:29:44 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:29:44.399+0000 7fd7d6194640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-13T23:29:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: pgmap v57: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-09-13T23:29:45.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:45.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:45.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:29:45.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:29:45.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:45.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:29:45.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:45.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:45.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:45.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T23:29:45.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T23:29:45.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:45 smithi076 ceph-mon[63913]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-13T23:29:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: pgmap v57: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-09-13T23:29:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:29:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:29:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:29:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T23:29:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T23:29:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[80039]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-13T23:29:45.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: pgmap v57: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-09-13T23:29:45.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:45.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:45.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:29:45.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:29:45.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:45.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:29:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T23:29:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T23:29:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:45 smithi022 ceph-mon[82397]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-13T23:29:46.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:46 smithi022 ceph-mon[80039]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded) 2024-09-13T23:29:46.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:46 smithi022 ceph-mon[80039]: Cluster is now healthy 2024-09-13T23:29:46.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:46 smithi022 ceph-mon[82397]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded) 2024-09-13T23:29:46.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:46 smithi022 ceph-mon[82397]: Cluster is now healthy 2024-09-13T23:29:46.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:46 smithi076 ceph-mon[63913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded) 2024-09-13T23:29:46.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:46 smithi076 ceph-mon[63913]: Cluster is now healthy 2024-09-13T23:29:47.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:47 smithi022 ceph-mon[80039]: pgmap v58: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 119 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:47.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:47 smithi022 ceph-mon[82397]: pgmap v58: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 119 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:47.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:47 smithi076 ceph-mon[63913]: pgmap v58: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 119 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:48.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:48 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:48.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:48 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:48.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:48 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:49.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:49 smithi022 ceph-mon[80039]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T23:29:49.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:49 smithi022 ceph-mon[82397]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T23:29:49.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:49 smithi076 ceph-mon[63913]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T23:29:51.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:51 smithi022 ceph-mon[82397]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:51.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:51 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:51.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:51 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:29:51.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:51 smithi022 ceph-mon[80039]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:51.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:51 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:51.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:51 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:29:51.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:51 smithi076 ceph-mon[63913]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:51.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:51 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:51.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:51 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:29:52.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:29:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:29:52] "GET /metrics HTTP/1.1" 200 38083 "" "Prometheus/2.43.0" 2024-09-13T23:29:53.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:53 smithi022 ceph-mon[80039]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:29:53.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:53 smithi022 ceph-mon[82397]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:29:53.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:53 smithi076 ceph-mon[63913]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:29:55.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:55 smithi022 ceph-mon[80039]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 878 B/s rd, 0 op/s 2024-09-13T23:29:55.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:55 smithi022 ceph-mon[82397]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 878 B/s rd, 0 op/s 2024-09-13T23:29:55.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:55 smithi076 ceph-mon[63913]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 878 B/s rd, 0 op/s 2024-09-13T23:29:56.409 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:29:57.096 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:29:57.096 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (17m) 16s ago 25m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:29:57.096 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (3m) 106s ago 24m 82.8M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:29:57.096 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (4m) 16s ago 24m 54.7M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:29:57.096 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (4m) 106s ago 29m 437M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:29:57.096 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (13m) 16s ago 31m 499M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:29:57.096 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (117s) 16s ago 31m 46.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:29:57.096 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (2m) 106s ago 30m 35.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:29:57.096 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (96s) 16s ago 30m 37.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (17m) 16s ago 25m 16.1M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (16m) 106s ago 25m 16.1M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (22s) 16s ago 29m 11.7M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (29m) 16s ago 29m 55.3M 2993M 17.2.0 e1d6a67b021e 217437057b8d 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (28m) 16s ago 28m 51.7M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (28m) 16s ago 28m 53.9M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (27m) 106s ago 27m 56.1M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (27m) 106s ago 27m 56.0M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (27m) 106s ago 27m 55.0M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (26m) 106s ago 26m 54.5M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (4m) 106s ago 25m 60.8M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (24m) 16s ago 24m 77.8M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:29:57.097 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (24m) 106s ago 24m 76.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:29:57.352 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:57 smithi022 ceph-mon[80039]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:57.352 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:57.352 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:57 smithi022 ceph-mon[80039]: from='client.54252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:57.352 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:57 smithi022 ceph-mon[80039]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:57.352 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:57 smithi022 ceph-mon[80039]: from='client.64198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:57.353 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:57 smithi022 ceph-mon[82397]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:57.353 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:57.353 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:57 smithi022 ceph-mon[82397]: from='client.54252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:57.353 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:57 smithi022 ceph-mon[82397]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:57.353 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:57 smithi022 ceph-mon[82397]: from='client.64198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:57.517 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:29:57.517 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:29:57.517 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:29:57.520 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-13T23:29:57.521 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 6 2024-09-13T23:29:57.522 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:29:57.522 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:29:57.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:57 smithi076 ceph-mon[63913]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:29:57.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:29:57.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:57 smithi076 ceph-mon[63913]: from='client.54252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:57.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:57 smithi076 ceph-mon[63913]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:57.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:57 smithi076 ceph-mon[63913]: from='client.64198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:57.869 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:29:57.869 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:29:57.869 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:29:57.869 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-09-13T23:29:57.869 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:29:57.869 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "1/8 daemons upgraded", 2024-09-13T23:29:57.869 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T23:29:57.869 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:29:57.870 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:29:58.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:58 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:58.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:58 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/68769683' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:58.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:58 smithi022 ceph-mon[80039]: from='client.54276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:58.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:58 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:58.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:58 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/68769683' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:58.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:58 smithi022 ceph-mon[82397]: from='client.54276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:58.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:58 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:29:58.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:58 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/68769683' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:29:58.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:58 smithi076 ceph-mon[63913]: from='client.54276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:29:59.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:29:59 smithi022 ceph-mon[80039]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:59.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:29:59 smithi022 ceph-mon[82397]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:29:59.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:29:59 smithi076 ceph-mon[63913]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:00.653 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:00 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T23:30:00.653 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:00 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T23:30:00.653 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:00 smithi022 ceph-mon[80039]: Upgrade: osd.1 is safe to restart 2024-09-13T23:30:00.653 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:00 smithi022 ceph-mon[80039]: overall HEALTH_OK 2024-09-13T23:30:00.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:00 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T23:30:00.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:00 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T23:30:00.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:00 smithi022 ceph-mon[82397]: Upgrade: osd.1 is safe to restart 2024-09-13T23:30:00.653 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:00 smithi022 ceph-mon[82397]: overall HEALTH_OK 2024-09-13T23:30:00.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:00 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T23:30:00.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:00 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T23:30:00.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:00 smithi076 ceph-mon[63913]: Upgrade: osd.1 is safe to restart 2024-09-13T23:30:00.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:00 smithi076 ceph-mon[63913]: overall HEALTH_OK 2024-09-13T23:30:01.985 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:01 smithi022 ceph-mon[80039]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:01.985 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:01 smithi022 ceph-mon[80039]: Upgrade: Updating osd.1 2024-09-13T23:30:01.985 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:01 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:01.985 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:01 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T23:30:01.985 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:01 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:01.985 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:01 smithi022 ceph-mon[80039]: Deploying daemon osd.1 on smithi022 2024-09-13T23:30:01.986 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:01 smithi022 ceph-mon[82397]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:01.986 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:01 smithi022 ceph-mon[82397]: Upgrade: Updating osd.1 2024-09-13T23:30:01.986 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:01 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:01.986 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:01 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T23:30:01.986 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:01 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:01.986 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:01 smithi022 ceph-mon[82397]: Deploying daemon osd.1 on smithi022 2024-09-13T23:30:02.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:01 smithi076 ceph-mon[63913]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:02.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:01 smithi076 ceph-mon[63913]: Upgrade: Updating osd.1 2024-09-13T23:30:02.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:01 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:02.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:01 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T23:30:02.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:01 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:02.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:01 smithi076 ceph-mon[63913]: Deploying daemon osd.1 on smithi022 2024-09-13T23:30:02.488 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:30:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:30:02] "GET /metrics HTTP/1.1" 200 38083 "" "Prometheus/2.43.0" 2024-09-13T23:30:03.868 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:03 smithi022 ceph-mon[80039]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:03.868 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:03 smithi022 ceph-mon[82397]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:03.868 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:03 smithi022 systemd[1]: Stopping Ceph osd.1 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:30:04.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:03 smithi076 ceph-mon[63913]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:04.176 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1[34222]: 2024-09-13T23:30:03.889+0000 7f816928d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:30:04.177 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1[34222]: 2024-09-13T23:30:03.889+0000 7f816928d700 -1 osd.1 118 *** Got signal Terminated *** 2024-09-13T23:30:04.177 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1[34222]: 2024-09-13T23:30:03.889+0000 7f816928d700 -1 osd.1 118 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:30:05.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:04 smithi076 ceph-mon[63913]: osd.1 marked itself down and dead 2024-09-13T23:30:05.155 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:04 smithi022 ceph-mon[80039]: osd.1 marked itself down and dead 2024-09-13T23:30:05.155 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:04 smithi022 ceph-mon[82397]: osd.1 marked itself down and dead 2024-09-13T23:30:05.426 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:05 smithi022 podman[94462]: 2024-09-13 23:30:05.154068365 +0000 UTC m=+1.365507843 container died 217437057b8da8cf0ff50cb084a663a4a590ca457ad19b0ff52ba9341f8990b4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, ceph=True, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_BRANCH=HEAD, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, name=centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, release=754, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.display-name=CentOS Stream 8) 2024-09-13T23:30:05.807 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:05 smithi022 ceph-mon[80039]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:05.807 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:05 smithi022 ceph-mon[80039]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:30:05.807 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:05 smithi022 ceph-mon[80039]: osdmap e119: 8 total, 7 up, 8 in 2024-09-13T23:30:05.807 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:05 smithi022 podman[94462]: 2024-09-13 23:30:05.546667415 +0000 UTC m=+1.758106888 container cleanup 217437057b8da8cf0ff50cb084a663a4a590ca457ad19b0ff52ba9341f8990b4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., ceph=True, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, distribution-scope=public, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-13T23:30:05.807 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:05 smithi022 bash[94462]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1 2024-09-13T23:30:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:05 smithi022 ceph-mon[82397]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:05 smithi022 ceph-mon[82397]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:30:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:05 smithi022 ceph-mon[82397]: osdmap e119: 8 total, 7 up, 8 in 2024-09-13T23:30:06.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:05 smithi076 ceph-mon[63913]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:06.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:05 smithi076 ceph-mon[63913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:30:06.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:05 smithi076 ceph-mon[63913]: osdmap e119: 8 total, 7 up, 8 in 2024-09-13T23:30:06.176 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:05 smithi022 podman[94488]: 2024-09-13 23:30:05.80642275 +0000 UTC m=+0.650878179 container remove 217437057b8da8cf0ff50cb084a663a4a590ca457ad19b0ff52ba9341f8990b4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, ceph=True, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, release=754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_CLEAN=True) 2024-09-13T23:30:06.494 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:06 smithi022 podman[94576]: 2024-09-13 23:30:06.082004081 +0000 UTC m=+0.016588226 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:30:06.494 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:06 smithi022 podman[94576]: 2024-09-13 23:30:06.182404394 +0000 UTC m=+0.116988601 container create 7f00df93ee4bc0267737702d822c738a2f3a47b8dbe54aed336d7a2007bb011d (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:30:06.494 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:06 smithi022 podman[94576]: 2024-09-13 23:30:06.401625067 +0000 UTC m=+0.336209203 container init 7f00df93ee4bc0267737702d822c738a2f3a47b8dbe54aed336d7a2007bb011d (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-13T23:30:06.494 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:06 smithi022 podman[94576]: 2024-09-13 23:30:06.405124948 +0000 UTC m=+0.339709079 container start 7f00df93ee4bc0267737702d822c738a2f3a47b8dbe54aed336d7a2007bb011d (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:30:06.494 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:06 smithi022 podman[94576]: 2024-09-13 23:30:06.493533062 +0000 UTC m=+0.428117204 container attach 7f00df93ee4bc0267737702d822c738a2f3a47b8dbe54aed336d7a2007bb011d (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T23:30:06.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:06 smithi022 ceph-mon[80039]: osdmap e120: 8 total, 7 up, 8 in 2024-09-13T23:30:06.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:06.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:30:06.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:06 smithi022 ceph-mon[82397]: osdmap e120: 8 total, 7 up, 8 in 2024-09-13T23:30:06.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:06.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:30:06.928 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:06 smithi022 podman[94576]: 2024-09-13 23:30:06.599781012 +0000 UTC m=+0.534365140 container died 7f00df93ee4bc0267737702d822c738a2f3a47b8dbe54aed336d7a2007bb011d (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-13T23:30:07.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:06 smithi076 ceph-mon[63913]: osdmap e120: 8 total, 7 up, 8 in 2024-09-13T23:30:07.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:07.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:30:07.650 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:07 smithi022 podman[94600]: 2024-09-13 23:30:07.197519237 +0000 UTC m=+0.591983401 container remove 7f00df93ee4bc0267737702d822c738a2f3a47b8dbe54aed336d7a2007bb011d (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-13T23:30:07.651 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:07 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-13T23:30:07.651 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:07 smithi022 systemd[1]: Stopped Ceph osd.1 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:30:07.651 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:07 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.1.service: Consumed 15.397s CPU time. 2024-09-13T23:30:07.651 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:07 smithi022 systemd[1]: Starting Ceph osd.1 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:30:07.910 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:07 smithi022 ceph-mon[80039]: pgmap v70: 161 pgs: 7 peering, 18 stale+active+clean, 136 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:30:07.910 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:07 smithi022 ceph-mon[80039]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T23:30:07.911 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:07 smithi022 podman[94700]: 2024-09-13 23:30:07.552747178 +0000 UTC m=+0.016554313 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:30:07.911 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:07 smithi022 podman[94700]: 2024-09-13 23:30:07.665415368 +0000 UTC m=+0.129222487 container create 8dd2a34d41ad1c0fc8e176428f61de11fc67ca708630731053cd3a56767f4444 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=) 2024-09-13T23:30:07.911 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:07 smithi022 ceph-mon[82397]: pgmap v70: 161 pgs: 7 peering, 18 stale+active+clean, 136 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:30:07.911 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:07 smithi022 ceph-mon[82397]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T23:30:08.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:07 smithi076 ceph-mon[63913]: pgmap v70: 161 pgs: 7 peering, 18 stale+active+clean, 136 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:30:08.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:07 smithi076 ceph-mon[63913]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T23:30:08.177 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:07 smithi022 podman[94700]: 2024-09-13 23:30:07.90981175 +0000 UTC m=+0.373618872 container init 8dd2a34d41ad1c0fc8e176428f61de11fc67ca708630731053cd3a56767f4444 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-13T23:30:08.177 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:07 smithi022 podman[94700]: 2024-09-13 23:30:07.913335915 +0000 UTC m=+0.377143043 container start 8dd2a34d41ad1c0fc8e176428f61de11fc67ca708630731053cd3a56767f4444 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T23:30:08.177 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:08 smithi022 podman[94700]: 2024-09-13 23:30:08.001498233 +0000 UTC m=+0.465305362 container attach 8dd2a34d41ad1c0fc8e176428f61de11fc67ca708630731053cd3a56767f4444 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:30:08.177 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:08 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate[94714]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:30:08.177 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:08 smithi022 bash[94700]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:30:08.177 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:08 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate[94714]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:30:08.178 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:08 smithi022 bash[94700]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:30:09.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:08 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:09.078 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:08 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:09.078 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:08 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:09.423 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate[94714]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T23:30:09.424 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 bash[94700]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T23:30:09.424 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate[94714]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T23:30:09.424 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 bash[94700]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T23:30:09.424 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate[94714]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T23:30:09.677 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 bash[94700]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T23:30:09.677 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate[94714]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T23:30:09.677 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 bash[94700]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T23:30:09.677 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate[94714]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T23:30:09.677 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 bash[94700]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T23:30:09.677 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate[94714]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T23:30:09.677 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 bash[94700]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T23:30:09.677 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate[94714]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-13T23:30:09.677 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 bash[94700]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-13T23:30:09.678 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:09 smithi022 podman[94700]: 2024-09-13 23:30:09.455899083 +0000 UTC m=+1.919706211 container died 8dd2a34d41ad1c0fc8e176428f61de11fc67ca708630731053cd3a56767f4444 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T23:30:10.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:09 smithi076 ceph-mon[63913]: pgmap v71: 161 pgs: 36 active+undersized, 7 peering, 20 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-09-13T23:30:10.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:09 smithi076 ceph-mon[63913]: Health check failed: Degraded data redundancy: 79/627 objects degraded (12.600%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T23:30:10.081 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:09 smithi022 ceph-mon[82397]: pgmap v71: 161 pgs: 36 active+undersized, 7 peering, 20 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-09-13T23:30:10.081 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:09 smithi022 ceph-mon[82397]: Health check failed: Degraded data redundancy: 79/627 objects degraded (12.600%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T23:30:10.082 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:09 smithi022 ceph-mon[80039]: pgmap v71: 161 pgs: 36 active+undersized, 7 peering, 20 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-09-13T23:30:10.082 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:09 smithi022 ceph-mon[80039]: Health check failed: Degraded data redundancy: 79/627 objects degraded (12.600%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T23:30:10.415 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:10 smithi022 podman[94846]: 2024-09-13 23:30:10.080766965 +0000 UTC m=+0.617572376 container remove 8dd2a34d41ad1c0fc8e176428f61de11fc67ca708630731053cd3a56767f4444 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-13T23:30:10.415 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:10 smithi022 podman[94908]: 2024-09-13 23:30:10.204526302 +0000 UTC m=+0.017122072 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:30:10.415 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:10 smithi022 podman[94908]: 2024-09-13 23:30:10.306797107 +0000 UTC m=+0.119392874 container create 866b71c23f0a390ff3d4a1ca0d2765f513d58c86dadccd8bc04480488906bc17 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T23:30:10.926 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:10 smithi022 podman[94908]: 2024-09-13 23:30:10.517583118 +0000 UTC m=+0.330178891 container init 866b71c23f0a390ff3d4a1ca0d2765f513d58c86dadccd8bc04480488906bc17 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T23:30:10.927 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:10 smithi022 podman[94908]: 2024-09-13 23:30:10.52102261 +0000 UTC m=+0.333618374 container start 866b71c23f0a390ff3d4a1ca0d2765f513d58c86dadccd8bc04480488906bc17 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-13T23:30:10.927 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:10 smithi022 ceph-osd[94927]: -- 172.21.15.22:0/923864589 <== mon.1 v2:172.21.15.22:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56362d55da40 con 0x56362e324400 2024-09-13T23:30:10.927 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:10 smithi022 bash[94908]: 866b71c23f0a390ff3d4a1ca0d2765f513d58c86dadccd8bc04480488906bc17 2024-09-13T23:30:10.927 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:10 smithi022 systemd[1]: Started Ceph osd.1 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:30:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:11 smithi076 ceph-mon[63913]: pgmap v72: 161 pgs: 36 active+undersized, 7 peering, 20 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-09-13T23:30:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:12.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:11 smithi022 ceph-mon[80039]: pgmap v72: 161 pgs: 36 active+undersized, 7 peering, 20 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-09-13T23:30:12.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:12.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:12.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:12.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:11 smithi022 ceph-mon[82397]: pgmap v72: 161 pgs: 36 active+undersized, 7 peering, 20 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-09-13T23:30:12.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:12.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:12.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:12.451 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1[94923]: 2024-09-13T23:30:12.181+0000 7f3e1a652640 -1 Falling back to public interface 2024-09-13T23:30:12.729 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:30:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:30:12] "GET /metrics HTTP/1.1" 200 38083 "" "Prometheus/2.43.0" 2024-09-13T23:30:12.995 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:12 smithi022 ceph-mon[80039]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T23:30:12.995 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:12 smithi022 ceph-mon[82397]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T23:30:13.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:12 smithi076 ceph-mon[63913]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T23:30:14.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:13 smithi076 ceph-mon[63913]: pgmap v73: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-09-13T23:30:14.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:13 smithi022 ceph-mon[82397]: pgmap v73: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-09-13T23:30:14.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:13 smithi022 ceph-mon[80039]: pgmap v73: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-09-13T23:30:16.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:15 smithi076 ceph-mon[63913]: pgmap v74: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-09-13T23:30:16.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:15 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 86/627 objects degraded (13.716%), 22 pgs degraded (PG_DEGRADED) 2024-09-13T23:30:16.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:15 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:16.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:15 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:16.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:15 smithi022 ceph-mon[82397]: pgmap v74: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-09-13T23:30:16.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:15 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 86/627 objects degraded (13.716%), 22 pgs degraded (PG_DEGRADED) 2024-09-13T23:30:16.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:15 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:16.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:15 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:16.177 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1[94923]: 2024-09-13T23:30:15.717+0000 7f3e1a652640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-13T23:30:16.177 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1[94923]: 2024-09-13T23:30:15.747+0000 7f3e1a652640 -1 osd.1 118 log_to_monitors true 2024-09-13T23:30:16.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:15 smithi022 ceph-mon[80039]: pgmap v74: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-09-13T23:30:16.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:15 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 86/627 objects degraded (13.716%), 22 pgs degraded (PG_DEGRADED) 2024-09-13T23:30:16.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:15 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:16.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:15 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:16.885 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:16 smithi022 ceph-mon[80039]: from='osd.1 [v2:172.21.15.22:6810/2366198677,v1:172.21.15.22:6811/2366198677]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T23:30:16.885 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:16 smithi022 ceph-mon[80039]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T23:30:16.885 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:16.885 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:16.886 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:16.886 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:16 smithi022 ceph-mon[82397]: from='osd.1 [v2:172.21.15.22:6810/2366198677,v1:172.21.15.22:6811/2366198677]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T23:30:16.886 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:16 smithi022 ceph-mon[82397]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T23:30:16.886 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:16.886 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:16.886 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:17.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:16 smithi076 ceph-mon[63913]: from='osd.1 [v2:172.21.15.22:6810/2366198677,v1:172.21.15.22:6811/2366198677]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T23:30:17.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:16 smithi076 ceph-mon[63913]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T23:30:17.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:17.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:17.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:17.176 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1[94923]: 2024-09-13T23:30:17.026+0000 7f3e0e3f5640 -1 osd.1 118 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:30:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-mon[82397]: pgmap v75: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-09-13T23:30:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-mon[82397]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T23:30:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-mon[82397]: osdmap e121: 8 total, 7 up, 8 in 2024-09-13T23:30:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-mon[82397]: from='osd.1 [v2:172.21.15.22:6810/2366198677,v1:172.21.15.22:6811/2366198677]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:30:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-mon[82397]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:30:17.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-mon[82397]: from='osd.1 ' entity='osd.1' 2024-09-13T23:30:17.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-mon[80039]: pgmap v75: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-09-13T23:30:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-mon[80039]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T23:30:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-mon[80039]: osdmap e121: 8 total, 7 up, 8 in 2024-09-13T23:30:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-mon[80039]: from='osd.1 [v2:172.21.15.22:6810/2366198677,v1:172.21.15.22:6811/2366198677]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:30:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-mon[80039]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:30:17.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:17 smithi022 ceph-mon[80039]: from='osd.1 ' entity='osd.1' 2024-09-13T23:30:18.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:17 smithi076 ceph-mon[63913]: pgmap v75: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-09-13T23:30:18.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:17 smithi076 ceph-mon[63913]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T23:30:18.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:17 smithi076 ceph-mon[63913]: osdmap e121: 8 total, 7 up, 8 in 2024-09-13T23:30:18.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:17 smithi076 ceph-mon[63913]: from='osd.1 [v2:172.21.15.22:6810/2366198677,v1:172.21.15.22:6811/2366198677]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:30:18.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:17 smithi076 ceph-mon[63913]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:30:18.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:17 smithi076 ceph-mon[63913]: from='osd.1 ' entity='osd.1' 2024-09-13T23:30:19.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:18 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:19.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:18 smithi076 ceph-mon[63913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:30:19.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:18 smithi076 ceph-mon[63913]: osd.1 [v2:172.21.15.22:6810/2366198677,v1:172.21.15.22:6811/2366198677] boot 2024-09-13T23:30:19.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:18 smithi076 ceph-mon[63913]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T23:30:19.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:18 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:30:19.069 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:18 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:19.069 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:18 smithi022 ceph-mon[82397]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:30:19.069 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:18 smithi022 ceph-mon[82397]: osd.1 [v2:172.21.15.22:6810/2366198677,v1:172.21.15.22:6811/2366198677] boot 2024-09-13T23:30:19.069 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:18 smithi022 ceph-mon[82397]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T23:30:19.069 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:18 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:30:19.070 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:18 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:19.070 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:18 smithi022 ceph-mon[80039]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:30:19.070 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:18 smithi022 ceph-mon[80039]: osd.1 [v2:172.21.15.22:6810/2366198677,v1:172.21.15.22:6811/2366198677] boot 2024-09-13T23:30:19.070 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:18 smithi022 ceph-mon[80039]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T23:30:19.070 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:18 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T23:30:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[82397]: pgmap v78: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 86/627 objects degraded (13.716%) 2024-09-13T23:30:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[82397]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T23:30:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:30:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:20.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[80039]: pgmap v78: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 86/627 objects degraded (13.716%) 2024-09-13T23:30:20.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[80039]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T23:30:20.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:20.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:20.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:20.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:30:20.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:20 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:20 smithi076 ceph-mon[63913]: pgmap v78: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 86/627 objects degraded (13.716%) 2024-09-13T23:30:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:20 smithi076 ceph-mon[63913]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T23:30:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:20 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:20 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:20.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:20 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:20.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:20 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:30:20.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:20 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:30:21.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:30:21.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: Upgrade: Finalizing container_image settings 2024-09-13T23:30:21.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:30:21.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:30:21.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:30:21.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:30:21.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:30:21.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:30:21.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:30:21.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:30:21.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:30:21.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:30:21.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:30:21.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:30:21.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:30:21.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:30:21.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: Upgrade: Complete! 2024-09-13T23:30:21.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:30:21.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:30:21.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:21.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:30:21.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:21.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:30:21.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:30:21.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:30:21.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: Upgrade: Finalizing container_image settings 2024-09-13T23:30:21.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:30:21.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:30:21.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:30:21.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:30:21.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:30:21.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:30:21.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:30:21.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:30:21.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:30:21.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:30:21.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:30:21.434 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:30:21.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:30:21.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.435 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:21.436 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: Upgrade: Finalizing container_image settings 2024-09-13T23:30:21.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:30:21.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:30:21.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:30:21.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:30:21.437 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:30:21.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:30:21.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:30:21.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:30:21.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:30:21.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:30:21.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:30:21.439 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:30:21.439 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:30:21.439 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:30:21.439 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.439 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.440 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.440 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.441 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.441 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.441 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: Upgrade: Complete! 2024-09-13T23:30:21.441 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:30:21.441 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:30:21.441 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:21.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:30:21.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:21.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:30:21.443 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:30:21.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:30:21.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:30:21.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:30:21.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: Upgrade: Complete! 2024-09-13T23:30:21.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:30:21.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:30:21.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:21.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:30:21.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:21.444 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:21.445 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:30:21.445 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:22.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:22 smithi022 ceph-mon[80039]: pgmap v80: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 86/627 objects degraded (13.716%) 2024-09-13T23:30:22.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:22.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:30:22.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:22 smithi022 ceph-mon[82397]: pgmap v80: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 86/627 objects degraded (13.716%) 2024-09-13T23:30:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:30:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:22.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:22 smithi076 ceph-mon[63913]: pgmap v80: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 86/627 objects degraded (13.716%) 2024-09-13T23:30:22.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:22.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:30:22.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:22.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:30:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:30:22] "GET /metrics HTTP/1.1" 200 38111 "" "Prometheus/2.43.0" 2024-09-13T23:30:23.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:23 smithi022 ceph-mon[80039]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 86/627 objects degraded (13.716%), 22 pgs degraded) 2024-09-13T23:30:23.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:23 smithi022 ceph-mon[80039]: Cluster is now healthy 2024-09-13T23:30:23.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:23 smithi022 ceph-mon[82397]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 86/627 objects degraded (13.716%), 22 pgs degraded) 2024-09-13T23:30:23.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:23 smithi022 ceph-mon[82397]: Cluster is now healthy 2024-09-13T23:30:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:23 smithi076 ceph-mon[63913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 86/627 objects degraded (13.716%), 22 pgs degraded) 2024-09-13T23:30:23.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:23 smithi076 ceph-mon[63913]: Cluster is now healthy 2024-09-13T23:30:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:24 smithi076 ceph-mon[63913]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T23:30:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:24 smithi022 ceph-mon[80039]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T23:30:24.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:24 smithi022 ceph-mon[82397]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T23:30:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:26 smithi076 ceph-mon[63913]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T23:30:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:26 smithi022 ceph-mon[80039]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T23:30:26.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:26 smithi022 ceph-mon[82397]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T23:30:28.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:28 smithi076 ceph-mon[63913]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-09-13T23:30:28.601 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:28 smithi022 ceph-mon[80039]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-09-13T23:30:28.602 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:28 smithi022 ceph-mon[82397]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-09-13T23:30:28.999 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:30:29.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:29 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:29.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:29 smithi022 ceph-mon[82397]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:29.314 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:29 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:29.314 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:29 smithi022 ceph-mon[80039]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:29 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:29.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:29 smithi076 ceph-mon[63913]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:30.320 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:30 smithi022 ceph-mon[80039]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:30:30.320 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:30 smithi022 ceph-mon[82397]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:30:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:30 smithi076 ceph-mon[63913]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:30:31.064 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:30:31.064 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (17m) 15s ago 25m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:30:31.064 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (4m) 2m ago 25m 82.8M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:30:31.064 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (4m) 15s ago 24m 56.5M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:30:31.064 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (4m) 2m ago 30m 437M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:30:31.064 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (14m) 15s ago 31m 500M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:30:31.064 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (2m) 15s ago 32m 47.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (3m) 2m ago 30m 35.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (2m) 15s ago 30m 39.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (17m) 15s ago 26m 15.9M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (17m) 2m ago 26m 16.1M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (56s) 15s ago 29m 49.7M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (20s) 15s ago 29m 12.1M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (29m) 15s ago 29m 51.8M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (28m) 15s ago 28m 54.4M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (28m) 2m ago 28m 56.1M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (28m) 2m ago 28m 56.0M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (27m) 2m ago 27m 55.0M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (27m) 2m ago 27m 54.5M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:30:31.065 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (4m) 2m ago 26m 60.8M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:30:31.066 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (24m) 15s ago 24m 78.0M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:30:31.066 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (24m) 2m ago 24m 76.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:30:31.896 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-09-13T23:30:32.453 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:32 smithi022 ceph-mon[80039]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 910 B/s rd, 0 op/s 2024-09-13T23:30:32.453 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:32 smithi022 ceph-mon[80039]: from='client.64228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:32.453 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:32 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:32.453 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:32 smithi022 ceph-mon[82397]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 910 B/s rd, 0 op/s 2024-09-13T23:30:32.454 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:32 smithi022 ceph-mon[82397]: from='client.64228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:32.454 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:32 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:32 smithi076 ceph-mon[63913]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 910 B/s rd, 0 op/s 2024-09-13T23:30:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:32 smithi076 ceph-mon[63913]: from='client.64228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:32 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:32.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:30:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:30:32] "GET /metrics HTTP/1.1" 200 38111 "" "Prometheus/2.43.0" 2024-09-13T23:30:34.038 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:30:34.402 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:34 smithi022 ceph-mon[80039]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:34.402 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:34 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/3204911775' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:34.402 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:34 smithi022 ceph-mon[82397]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:34.403 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:34 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/3204911775' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:34.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:34 smithi076 ceph-mon[63913]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:34.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:34 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/3204911775' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:34.727 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-09-13T23:30:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:36 smithi022 ceph-mon[80039]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:36.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:36 smithi022 ceph-mon[82397]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:36 smithi076 ceph-mon[63913]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:37.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:37 smithi022 ceph-mon[80039]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:37.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:37.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:30:37.495 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:37 smithi022 ceph-mon[80039]: from='client.64240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:37.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:37 smithi022 ceph-mon[82397]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:37.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:37.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:30:37.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:37 smithi022 ceph-mon[82397]: from='client.64240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:37.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:37 smithi076 ceph-mon[63913]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:37.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:37.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:30:37.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:37 smithi076 ceph-mon[63913]: from='client.64240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:38.113 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:30:38.398 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:38 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:38.398 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:38 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:38.765 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T23:30:38.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:38 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:39.385 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:39 smithi022 ceph-mon[80039]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:39.386 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:39 smithi022 ceph-mon[82397]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:39.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:39 smithi076 ceph-mon[63913]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:40.831 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:30:40.831 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": null, 2024-09-13T23:30:40.831 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": false, 2024-09-13T23:30:40.831 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "", 2024-09-13T23:30:40.832 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:30:40.832 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": null, 2024-09-13T23:30:40.832 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-09-13T23:30:40.832 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:30:40.832 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:30:41.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:41 smithi022 ceph-mon[80039]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:41.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:41 smithi022 ceph-mon[80039]: from='client.54312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:41.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:41 smithi022 ceph-mon[82397]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:41.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:41 smithi022 ceph-mon[82397]: from='client.54312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:41.524 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T23:30:41.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:41 smithi076 ceph-mon[63913]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:41.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:41 smithi076 ceph-mon[63913]: from='client.54312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:42.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:30:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:30:42] "GET /metrics HTTP/1.1" 200 38103 "" "Prometheus/2.43.0" 2024-09-13T23:30:43.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:43 smithi022 ceph-mon[80039]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:43.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:43 smithi022 ceph-mon[82397]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:43.734 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-09-13T23:30:43.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:43 smithi076 ceph-mon[63913]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:44.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:44 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/939929271' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:30:44.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:44 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/939929271' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:30:44.501 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-09-13T23:30:44.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:44 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/939929271' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:30:45.429 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:45 smithi022 ceph-mon[80039]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:45.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:45 smithi022 ceph-mon[82397]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:45.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:45 smithi076 ceph-mon[63913]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:47.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:47 smithi022 ceph-mon[80039]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:47.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:47 smithi022 ceph-mon[80039]: from='client.64258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:47.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:47 smithi022 ceph-mon[82397]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:47.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:47 smithi022 ceph-mon[82397]: from='client.64258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:47.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:47 smithi076 ceph-mon[63913]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:47.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:47 smithi076 ceph-mon[63913]: from='client.64258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:47.799 INFO:teuthology.orchestra.run.smithi022.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:30:48.442 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T23:30:48.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:48 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:48.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:48 smithi022 ceph-mon[80039]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:30:48.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:48 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:48.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:48 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:48.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:48 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:30:48.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:48 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:48.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:48 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:48.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:48 smithi022 ceph-mon[82397]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:30:48.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:48 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:48.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:48 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:48.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:48 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:30:48.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:48 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:49.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:48 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:49.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:48 smithi076 ceph-mon[63913]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:30:49.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:48 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:49.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:48 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:49.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:48 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:30:49.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:48 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[82397]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:50.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[82397]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:30:50.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:30:50.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:50.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:50.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:50.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T23:30:50.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[80039]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:50.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[80039]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:30:50.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:30:50.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:50.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:50.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:50.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:50 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T23:30:50.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:50 smithi076 ceph-mon[63913]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:50.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:50 smithi076 ceph-mon[63913]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:30:50.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:50 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:50 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:30:50.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:50 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:50.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:50 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:50 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:50.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:50 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:50 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:50.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:50 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:50.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:50 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T23:30:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[82397]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:30:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[82397]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:30:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mgr 2024-09-13T23:30:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mon 2024-09-13T23:30:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all crash 2024-09-13T23:30:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T23:30:51.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[82397]: Upgrade: osd.2 is safe to restart 2024-09-13T23:30:51.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[80039]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:30:51.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[80039]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:30:51.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mgr 2024-09-13T23:30:51.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mon 2024-09-13T23:30:51.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all crash 2024-09-13T23:30:51.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T23:30:51.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:51 smithi022 ceph-mon[80039]: Upgrade: osd.2 is safe to restart 2024-09-13T23:30:51.529 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:30:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:51 smithi076 ceph-mon[63913]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:30:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:51 smithi076 ceph-mon[63913]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:30:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:51 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mgr 2024-09-13T23:30:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:51 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mon 2024-09-13T23:30:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:51 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all crash 2024-09-13T23:30:51.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:51 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T23:30:51.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:51 smithi076 ceph-mon[63913]: Upgrade: osd.2 is safe to restart 2024-09-13T23:30:52.222 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:30:52.222 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (17m) 36s ago 25m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:30:52.222 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (4m) 2m ago 25m 82.8M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:30:52.222 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (5m) 36s ago 25m 56.5M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:30:52.222 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (4m) 2m ago 30m 437M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:30:52.222 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (14m) 36s ago 32m 500M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:30:52.222 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (2m) 36s ago 32m 47.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:30:52.222 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (3m) 2m ago 30m 35.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (2m) 36s ago 31m 39.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (18m) 36s ago 26m 15.9M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (17m) 2m ago 26m 16.1M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (77s) 36s ago 30m 49.7M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (42s) 36s ago 29m 12.1M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (29m) 36s ago 29m 51.8M 2993M 17.2.0 e1d6a67b021e 12647567931c 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (29m) 36s ago 29m 54.4M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (28m) 2m ago 28m 56.1M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (28m) 2m ago 28m 56.0M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (27m) 2m ago 27m 55.0M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (27m) 2m ago 27m 54.5M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (5m) 2m ago 26m 60.8M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:30:52.223 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (25m) 36s ago 25m 78.0M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:30:52.224 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (25m) 2m ago 25m 76.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:30:52.335 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:52 smithi022 ceph-mon[80039]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:52.335 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:52 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:30:52.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:52 smithi022 ceph-mon[82397]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:52.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:52 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:30:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:52 smithi076 ceph-mon[63913]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:52.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:52 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:30:52.664 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:30:52.664 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:30:52.664 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:30:52.664 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:30:52.664 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:30:52.664 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:30:52.664 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:30:52.664 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:30:52.664 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-13T23:30:52.665 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:30:52.665 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:30:52.665 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:30:52.665 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:30:52.665 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:30:52.665 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:30:52.665 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-13T23:30:52.665 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 7 2024-09-13T23:30:52.665 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:30:52.665 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:30:52.666 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:30:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:30:52] "GET /metrics HTTP/1.1" 200 38104 "" "Prometheus/2.43.0" 2024-09-13T23:30:53.018 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:30:53.018 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:30:53.018 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:30:53.019 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-09-13T23:30:53.019 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:30:53.019 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "2/8 daemons upgraded", 2024-09-13T23:30:53.019 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T23:30:53.019 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:30:53.019 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:30:53.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[80039]: from='client.64264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:53.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[80039]: from='client.54336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:53.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[80039]: from='client.64276 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:53.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:53.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T23:30:53.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:53.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/581555082' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:53.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[82397]: from='client.64264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:53.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[82397]: from='client.54336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:53.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[82397]: from='client.64276 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:53.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:53.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T23:30:53.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:53.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:53 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/581555082' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:53.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:53 smithi076 ceph-mon[63913]: from='client.64264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:53.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:53 smithi076 ceph-mon[63913]: from='client.54336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:53.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:53 smithi076 ceph-mon[63913]: from='client.64276 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:53.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:53 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:30:53.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:53 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T23:30:53.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:53 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:30:53.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:53 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/581555082' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:30:54.353 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:54 smithi022 ceph-mon[80039]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:54.353 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:54 smithi022 ceph-mon[80039]: Upgrade: Updating osd.2 2024-09-13T23:30:54.353 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:54 smithi022 ceph-mon[80039]: Deploying daemon osd.2 on smithi022 2024-09-13T23:30:54.353 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:54 smithi022 ceph-mon[80039]: from='client.54354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:54.354 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:54 smithi022 ceph-mon[82397]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:54.354 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:54 smithi022 ceph-mon[82397]: Upgrade: Updating osd.2 2024-09-13T23:30:54.354 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:54 smithi022 ceph-mon[82397]: Deploying daemon osd.2 on smithi022 2024-09-13T23:30:54.354 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:54 smithi022 ceph-mon[82397]: from='client.54354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:54 smithi076 ceph-mon[63913]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:30:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:54 smithi076 ceph-mon[63913]: Upgrade: Updating osd.2 2024-09-13T23:30:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:54 smithi076 ceph-mon[63913]: Deploying daemon osd.2 on smithi022 2024-09-13T23:30:54.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:54 smithi076 ceph-mon[63913]: from='client.54354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:30:55.590 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:55 smithi022 systemd[1]: Stopping Ceph osd.2 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:30:55.926 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2[37137]: 2024-09-13T23:30:55.588+0000 7f9cd37b9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:30:55.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2[37137]: 2024-09-13T23:30:55.588+0000 7f9cd37b9700 -1 osd.2 123 *** Got signal Terminated *** 2024-09-13T23:30:55.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2[37137]: 2024-09-13T23:30:55.588+0000 7f9cd37b9700 -1 osd.2 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:30:56.532 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:56 smithi022 ceph-mon[80039]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:56.532 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:56 smithi022 ceph-mon[80039]: osd.2 marked itself down and dead 2024-09-13T23:30:56.533 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:56 smithi022 ceph-mon[82397]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:56.533 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:56 smithi022 ceph-mon[82397]: osd.2 marked itself down and dead 2024-09-13T23:30:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:56 smithi076 ceph-mon[63913]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:30:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:56 smithi076 ceph-mon[63913]: osd.2 marked itself down and dead 2024-09-13T23:30:56.840 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:56 smithi022 podman[101004]: 2024-09-13 23:30:56.53151486 +0000 UTC m=+1.077322373 container died 12647567931cc24634f59cd7c35cea15a722df282e72fa26bbf6275181409597 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, RELEASE=HEAD, distribution-scope=public, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, version=8, GIT_BRANCH=HEAD) 2024-09-13T23:30:57.172 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:56 smithi022 podman[101004]: 2024-09-13 23:30:56.920831207 +0000 UTC m=+1.466638754 container cleanup 12647567931cc24634f59cd7c35cea15a722df282e72fa26bbf6275181409597 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, ceph=True, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, architecture=x86_64, io.openshift.expose-services=, version=8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-13T23:30:57.173 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:56 smithi022 bash[101004]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2 2024-09-13T23:30:57.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:57 smithi022 ceph-mon[82397]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:30:57.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:57 smithi022 ceph-mon[82397]: osdmap e124: 8 total, 7 up, 8 in 2024-09-13T23:30:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:57 smithi022 ceph-mon[80039]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:30:57.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:57 smithi022 ceph-mon[80039]: osdmap e124: 8 total, 7 up, 8 in 2024-09-13T23:30:57.427 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:57 smithi022 podman[101021]: 2024-09-13 23:30:57.171902322 +0000 UTC m=+0.632593561 container remove 12647567931cc24634f59cd7c35cea15a722df282e72fa26bbf6275181409597 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, architecture=x86_64, GIT_CLEAN=True, ceph=True, vcs-type=git, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux ) 2024-09-13T23:30:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:57 smithi076 ceph-mon[63913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:30:57.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:57 smithi076 ceph-mon[63913]: osdmap e124: 8 total, 7 up, 8 in 2024-09-13T23:30:57.793 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:57 smithi022 podman[101110]: 2024-09-13 23:30:57.412403975 +0000 UTC m=+0.016650053 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:30:57.793 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:57 smithi022 podman[101110]: 2024-09-13 23:30:57.57316663 +0000 UTC m=+0.177412744 container create f09f859c260d2b3dad6d12e468d69d9daf7ce76d884e0ce87682ce6498f846f1 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:30:58.177 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:57 smithi022 podman[101110]: 2024-09-13 23:30:57.792626068 +0000 UTC m=+0.396872146 container init f09f859c260d2b3dad6d12e468d69d9daf7ce76d884e0ce87682ce6498f846f1 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-13T23:30:58.177 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:57 smithi022 podman[101110]: 2024-09-13 23:30:57.796378048 +0000 UTC m=+0.400624118 container start f09f859c260d2b3dad6d12e468d69d9daf7ce76d884e0ce87682ce6498f846f1 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T23:30:58.177 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:57 smithi022 podman[101110]: 2024-09-13 23:30:57.875539016 +0000 UTC m=+0.479785093 container attach f09f859c260d2b3dad6d12e468d69d9daf7ce76d884e0ce87682ce6498f846f1 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T23:30:58.177 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:57 smithi022 podman[101110]: 2024-09-13 23:30:57.991485034 +0000 UTC m=+0.595731103 container died f09f859c260d2b3dad6d12e468d69d9daf7ce76d884e0ce87682ce6498f846f1 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T23:30:58.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:58 smithi076 ceph-mon[63913]: pgmap v99: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:30:58.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:58 smithi076 ceph-mon[63913]: osdmap e125: 8 total, 7 up, 8 in 2024-09-13T23:30:58.589 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:58 smithi022 ceph-mon[80039]: pgmap v99: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:30:58.589 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:58 smithi022 ceph-mon[80039]: osdmap e125: 8 total, 7 up, 8 in 2024-09-13T23:30:58.589 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:58 smithi022 ceph-mon[82397]: pgmap v99: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:30:58.589 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:58 smithi022 ceph-mon[82397]: osdmap e125: 8 total, 7 up, 8 in 2024-09-13T23:30:58.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:58 smithi022 podman[101133]: 2024-09-13 23:30:58.587939984 +0000 UTC m=+0.586325058 container remove f09f859c260d2b3dad6d12e468d69d9daf7ce76d884e0ce87682ce6498f846f1 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:30:58.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:58 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-13T23:30:58.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:58 smithi022 systemd[1]: Stopped Ceph osd.2 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:30:58.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:58 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.2.service: Consumed 13.001s CPU time. 2024-09-13T23:30:58.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:58 smithi022 systemd[1]: Starting Ceph osd.2 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:30:59.317 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:30:59 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:59.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:30:59 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:59.318 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:59 smithi022 podman[101237]: 2024-09-13 23:30:58.937244447 +0000 UTC m=+0.016418494 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:30:59.318 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:59 smithi022 podman[101237]: 2024-09-13 23:30:59.055875596 +0000 UTC m=+0.135049634 container create 1522155e04f86b0a21f616df14bf3153801147a5d2466b9e8ccf8c1ab4680ec5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:30:59.318 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:59 smithi022 podman[101237]: 2024-09-13 23:30:59.316831996 +0000 UTC m=+0.396006029 container init 1522155e04f86b0a21f616df14bf3153801147a5d2466b9e8ccf8c1ab4680ec5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-13T23:30:59.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:30:59 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:30:59.677 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:59 smithi022 podman[101237]: 2024-09-13 23:30:59.320708404 +0000 UTC m=+0.399882446 container start 1522155e04f86b0a21f616df14bf3153801147a5d2466b9e8ccf8c1ab4680ec5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-13T23:30:59.677 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:59 smithi022 podman[101237]: 2024-09-13 23:30:59.400274174 +0000 UTC m=+0.479448207 container attach 1522155e04f86b0a21f616df14bf3153801147a5d2466b9e8ccf8c1ab4680ec5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T23:30:59.678 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:59 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate[101252]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:30:59.678 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:59 smithi022 bash[101237]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:30:59.678 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:59 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate[101252]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:30:59.678 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:30:59 smithi022 bash[101237]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:31:00.535 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-mon[82397]: pgmap v101: 161 pgs: 23 active+undersized, 6 peering, 2 stale+active+clean, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 32/627 objects degraded (5.104%) 2024-09-13T23:31:00.535 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-mon[82397]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T23:31:00.535 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-mon[82397]: Health check failed: Degraded data redundancy: 32/627 objects degraded (5.104%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T23:31:00.536 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-mon[80039]: pgmap v101: 161 pgs: 23 active+undersized, 6 peering, 2 stale+active+clean, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 32/627 objects degraded (5.104%) 2024-09-13T23:31:00.536 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-mon[80039]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T23:31:00.536 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-mon[80039]: Health check failed: Degraded data redundancy: 32/627 objects degraded (5.104%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T23:31:00.536 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate[101252]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T23:31:00.536 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 bash[101237]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T23:31:00.536 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate[101252]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T23:31:00.536 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 bash[101237]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T23:31:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:00 smithi076 ceph-mon[63913]: pgmap v101: 161 pgs: 23 active+undersized, 6 peering, 2 stale+active+clean, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 32/627 objects degraded (5.104%) 2024-09-13T23:31:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:00 smithi076 ceph-mon[63913]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T23:31:00.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:00 smithi076 ceph-mon[63913]: Health check failed: Degraded data redundancy: 32/627 objects degraded (5.104%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T23:31:00.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate[101252]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T23:31:00.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 bash[101237]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T23:31:00.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate[101252]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T23:31:00.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 bash[101237]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T23:31:00.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate[101252]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T23:31:00.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 bash[101237]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T23:31:00.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate[101252]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T23:31:00.928 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 bash[101237]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T23:31:00.928 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate[101252]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-13T23:31:00.928 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 bash[101237]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-13T23:31:00.928 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:00 smithi022 podman[101237]: 2024-09-13 23:31:00.567412844 +0000 UTC m=+1.646586896 container died 1522155e04f86b0a21f616df14bf3153801147a5d2466b9e8ccf8c1ab4680ec5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2) 2024-09-13T23:31:01.591 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:01 smithi022 podman[101392]: 2024-09-13 23:31:01.254329057 +0000 UTC m=+0.680212198 container remove 1522155e04f86b0a21f616df14bf3153801147a5d2466b9e8ccf8c1ab4680ec5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:31:01.591 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:01 smithi022 podman[101452]: 2024-09-13 23:31:01.471937501 +0000 UTC m=+0.111627484 container create 42db9ab27cb17f01e810b63b9045f918477be45876dc3bcc98ef99326601c33b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T23:31:01.591 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:01 smithi022 podman[101452]: 2024-09-13 23:31:01.377464177 +0000 UTC m=+0.017154132 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:31:02.177 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:01 smithi022 podman[101452]: 2024-09-13 23:31:01.699634 +0000 UTC m=+0.339323944 container init 42db9ab27cb17f01e810b63b9045f918477be45876dc3bcc98ef99326601c33b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:31:02.177 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:01 smithi022 podman[101452]: 2024-09-13 23:31:01.703102976 +0000 UTC m=+0.342792927 container start 42db9ab27cb17f01e810b63b9045f918477be45876dc3bcc98ef99326601c33b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-13T23:31:02.177 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:01 smithi022 ceph-osd[101471]: -- 172.21.15.22:0/3663459919 <== mon.1 v2:172.21.15.22:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56487c47ba40 con 0x56487d242400 2024-09-13T23:31:02.177 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:01 smithi022 bash[101452]: 42db9ab27cb17f01e810b63b9045f918477be45876dc3bcc98ef99326601c33b 2024-09-13T23:31:02.177 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:01 smithi022 systemd[1]: Started Ceph osd.2 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:31:02.514 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:31:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:31:02] "GET /metrics HTTP/1.1" 200 38104 "" "Prometheus/2.43.0" 2024-09-13T23:31:02.514 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:02 smithi022 ceph-mon[80039]: pgmap v102: 161 pgs: 23 active+undersized, 6 peering, 2 stale+active+clean, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 32/627 objects degraded (5.104%) 2024-09-13T23:31:02.514 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:02 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:02.514 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:02 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:02.514 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:02 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:02.515 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:02 smithi022 ceph-mon[82397]: pgmap v102: 161 pgs: 23 active+undersized, 6 peering, 2 stale+active+clean, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 32/627 objects degraded (5.104%) 2024-09-13T23:31:02.515 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:02 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:02.515 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:02 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:02.515 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:02 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:02.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:02 smithi076 ceph-mon[63913]: pgmap v102: 161 pgs: 23 active+undersized, 6 peering, 2 stale+active+clean, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 32/627 objects degraded (5.104%) 2024-09-13T23:31:02.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:02 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:02.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:02 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:02.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:02 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:03.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:03 smithi076 ceph-mon[63913]: pgmap v103: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:03.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:03 smithi076 ceph-mon[63913]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T23:31:03.563 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:03 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2[101467]: 2024-09-13T23:31:03.350+0000 7f4822137640 -1 Falling back to public interface 2024-09-13T23:31:03.563 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:03 smithi022 ceph-mon[82397]: pgmap v103: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:03.563 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:03 smithi022 ceph-mon[82397]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T23:31:03.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:03 smithi022 ceph-mon[80039]: pgmap v103: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:03.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:03 smithi022 ceph-mon[80039]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T23:31:05.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:05 smithi022 ceph-mon[82397]: pgmap v104: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 379 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:05.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:05 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 58/627 objects degraded (9.250%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T23:31:05.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:05 smithi022 ceph-mon[80039]: pgmap v104: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 379 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:05.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:05 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 58/627 objects degraded (9.250%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T23:31:05.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:05 smithi076 ceph-mon[63913]: pgmap v104: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 379 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:05.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:05 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 58/627 objects degraded (9.250%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T23:31:06.642 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:06 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2[101467]: 2024-09-13T23:31:06.622+0000 7f4822137640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-13T23:31:06.926 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:06 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2[101467]: 2024-09-13T23:31:06.641+0000 7f4822137640 -1 osd.2 123 log_to_monitors true 2024-09-13T23:31:07.628 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[82397]: pgmap v105: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:07.628 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:07.628 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:31:07.628 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[82397]: from='osd.2 [v2:172.21.15.22:6818/911243162,v1:172.21.15.22:6819/911243162]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T23:31:07.628 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[82397]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T23:31:07.629 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:07.629 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:07.629 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[80039]: pgmap v105: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:07.629 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:07.629 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:31:07.629 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[80039]: from='osd.2 [v2:172.21.15.22:6818/911243162,v1:172.21.15.22:6819/911243162]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T23:31:07.629 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[80039]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T23:31:07.629 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:07.629 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:07.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:07 smithi076 ceph-mon[63913]: pgmap v105: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:07.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:07.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:31:07.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:07 smithi076 ceph-mon[63913]: from='osd.2 [v2:172.21.15.22:6818/911243162,v1:172.21.15.22:6819/911243162]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T23:31:07.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:07 smithi076 ceph-mon[63913]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T23:31:07.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:07.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:07.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:31:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2[101467]: 2024-09-13T23:31:07.627+0000 7f4815eda640 -1 osd.2 123 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:31:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[80039]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T23:31:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[80039]: osdmap e126: 8 total, 7 up, 8 in 2024-09-13T23:31:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[80039]: from='osd.2 [v2:172.21.15.22:6818/911243162,v1:172.21.15.22:6819/911243162]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:31:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[80039]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:31:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:08.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:08.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[80039]: from='osd.2 ' entity='osd.2' 2024-09-13T23:31:08.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[82397]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T23:31:08.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[82397]: osdmap e126: 8 total, 7 up, 8 in 2024-09-13T23:31:08.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[82397]: from='osd.2 [v2:172.21.15.22:6818/911243162,v1:172.21.15.22:6819/911243162]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:31:08.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[82397]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:31:08.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:08.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:08.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:08.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:08 smithi022 ceph-mon[82397]: from='osd.2 ' entity='osd.2' 2024-09-13T23:31:08.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:08 smithi076 ceph-mon[63913]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T23:31:08.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:08 smithi076 ceph-mon[63913]: osdmap e126: 8 total, 7 up, 8 in 2024-09-13T23:31:08.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:08 smithi076 ceph-mon[63913]: from='osd.2 [v2:172.21.15.22:6818/911243162,v1:172.21.15.22:6819/911243162]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:31:08.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:08 smithi076 ceph-mon[63913]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:31:08.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:08 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:08.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:08 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:08.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:08 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:08.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:08 smithi076 ceph-mon[63913]: from='osd.2 ' entity='osd.2' 2024-09-13T23:31:09.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:09 smithi022 ceph-mon[80039]: pgmap v107: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:09.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:09 smithi022 ceph-mon[80039]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:31:09.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:09 smithi022 ceph-mon[80039]: osd.2 [v2:172.21.15.22:6818/911243162,v1:172.21.15.22:6819/911243162] boot 2024-09-13T23:31:09.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:09 smithi022 ceph-mon[80039]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T23:31:09.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:09 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:31:09.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:09 smithi022 ceph-mon[82397]: pgmap v107: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:09.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:09 smithi022 ceph-mon[82397]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:31:09.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:09 smithi022 ceph-mon[82397]: osd.2 [v2:172.21.15.22:6818/911243162,v1:172.21.15.22:6819/911243162] boot 2024-09-13T23:31:09.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:09 smithi022 ceph-mon[82397]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T23:31:09.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:09 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:31:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:09 smithi076 ceph-mon[63913]: pgmap v107: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:09 smithi076 ceph-mon[63913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:31:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:09 smithi076 ceph-mon[63913]: osd.2 [v2:172.21.15.22:6818/911243162,v1:172.21.15.22:6819/911243162] boot 2024-09-13T23:31:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:09 smithi076 ceph-mon[63913]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T23:31:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:09 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T23:31:10.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:10 smithi022 ceph-mon[80039]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T23:31:10.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:10 smithi022 ceph-mon[82397]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T23:31:11.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:10 smithi076 ceph-mon[63913]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T23:31:12.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: pgmap v110: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:12.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:12.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:31:12.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: Upgrade: Finalizing container_image settings 2024-09-13T23:31:12.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:31:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:31:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:31:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:31:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:31:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:31:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:31:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:31:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:31:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:31:12.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:31:12.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:31:12.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:31:12.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:31:12.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: Upgrade: Complete! 2024-09-13T23:31:12.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:31:12.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:31:12.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:12.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:31:12.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:12.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:31:12.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:11 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: pgmap v110: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:12.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:12.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:31:12.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:31:12.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:31:12.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: Upgrade: Finalizing container_image settings 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:31:12.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:31:12.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: pgmap v110: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 58/627 objects degraded (9.250%) 2024-09-13T23:31:12.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:12.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:31:12.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.180 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: Upgrade: Finalizing container_image settings 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:31:12.181 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:31:12.182 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: Upgrade: Complete! 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:31:12.183 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:12.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:31:12.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.184 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:31:12.184 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:31:12.184 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.184 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:31:12.184 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:31:12.184 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:31:12.184 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: Upgrade: Complete! 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:31:12.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:12.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:31:12.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:11 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:12.817 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:31:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:31:12] "GET /metrics HTTP/1.1" 200 38108 "" "Prometheus/2.43.0" 2024-09-13T23:31:13.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:12 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 3/627 objects degraded (0.478%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T23:31:13.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:12 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 3/627 objects degraded (0.478%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T23:31:13.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:12 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 3/627 objects degraded (0.478%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T23:31:14.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:13 smithi022 ceph-mon[80039]: pgmap v111: 161 pgs: 11 active+undersized, 2 active+undersized+degraded, 148 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 3/627 objects degraded (0.478%) 2024-09-13T23:31:14.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:13 smithi022 ceph-mon[82397]: pgmap v111: 161 pgs: 11 active+undersized, 2 active+undersized+degraded, 148 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 3/627 objects degraded (0.478%) 2024-09-13T23:31:14.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:13 smithi076 ceph-mon[63913]: pgmap v111: 161 pgs: 11 active+undersized, 2 active+undersized+degraded, 148 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 3/627 objects degraded (0.478%) 2024-09-13T23:31:15.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:14 smithi022 ceph-mon[80039]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/627 objects degraded (0.478%), 2 pgs degraded) 2024-09-13T23:31:15.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:14 smithi022 ceph-mon[80039]: Cluster is now healthy 2024-09-13T23:31:15.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:14 smithi022 ceph-mon[82397]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/627 objects degraded (0.478%), 2 pgs degraded) 2024-09-13T23:31:15.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:14 smithi022 ceph-mon[82397]: Cluster is now healthy 2024-09-13T23:31:15.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:14 smithi076 ceph-mon[63913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/627 objects degraded (0.478%), 2 pgs degraded) 2024-09-13T23:31:15.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:14 smithi076 ceph-mon[63913]: Cluster is now healthy 2024-09-13T23:31:16.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:15 smithi022 ceph-mon[80039]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 737 B/s rd, 0 op/s 2024-09-13T23:31:16.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:15 smithi022 ceph-mon[82397]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 737 B/s rd, 0 op/s 2024-09-13T23:31:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:15 smithi076 ceph-mon[63913]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 737 B/s rd, 0 op/s 2024-09-13T23:31:18.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:17 smithi022 ceph-mon[82397]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-13T23:31:18.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:17 smithi022 ceph-mon[80039]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-13T23:31:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:17 smithi076 ceph-mon[63913]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-13T23:31:19.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:18 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:19.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:18 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:19.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:18 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:20.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:19 smithi022 ceph-mon[80039]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T23:31:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:19 smithi022 ceph-mon[82397]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T23:31:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:19 smithi076 ceph-mon[63913]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T23:31:22.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:21 smithi022 ceph-mon[80039]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 960 B/s rd, 0 op/s 2024-09-13T23:31:22.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:22.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:31:22.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:21 smithi022 ceph-mon[82397]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 960 B/s rd, 0 op/s 2024-09-13T23:31:22.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:22.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:31:22.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:21 smithi076 ceph-mon[63913]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 960 B/s rd, 0 op/s 2024-09-13T23:31:22.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:22.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:31:22.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:31:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:31:22] "GET /metrics HTTP/1.1" 200 38112 "" "Prometheus/2.43.0" 2024-09-13T23:31:24.000 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:23 smithi022 ceph-mon[80039]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:24.000 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:23 smithi022 ceph-mon[82397]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:24.024 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:31:24.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:23 smithi076 ceph-mon[63913]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:25.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:24 smithi022 ceph-mon[80039]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:25.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:24 smithi022 ceph-mon[82397]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:25.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:24 smithi076 ceph-mon[63913]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:26.063 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:31:26.063 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (18m) 19s ago 26m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:31:26.063 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (5m) 3m ago 26m 82.8M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:31:26.063 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (5m) 19s ago 25m 58.1M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:31:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (5m) 3m ago 31m 437M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:31:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (15m) 19s ago 32m 501M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:31:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (3m) 19s ago 32m 51.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:31:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (4m) 3m ago 31m 35.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:31:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (3m) 19s ago 31m 39.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:31:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (18m) 19s ago 27m 16.4M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:31:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (18m) 3m ago 27m 16.1M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:31:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (111s) 19s ago 30m 51.3M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:31:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (75s) 19s ago 30m 52.2M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:31:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (24s) 19s ago 30m 11.7M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 42db9ab27cb1 2024-09-13T23:31:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (29m) 19s ago 29m 56.1M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:31:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (29m) 3m ago 29m 56.1M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:31:26.065 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (28m) 3m ago 28m 56.0M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:31:26.065 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (28m) 3m ago 28m 55.0M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:31:26.065 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (28m) 3m ago 28m 54.5M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:31:26.065 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (5m) 3m ago 27m 60.8M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:31:26.065 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (25m) 19s ago 25m 78.2M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:31:26.065 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (25m) 3m ago 25m 76.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:31:26.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:25 smithi022 ceph-mon[82397]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:26.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:25 smithi022 ceph-mon[80039]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:25 smithi076 ceph-mon[63913]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:26.894 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-09-13T23:31:27.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:26 smithi022 ceph-mon[80039]: from='client.54372 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:27.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:26 smithi022 ceph-mon[82397]: from='client.54372 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:27.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:26 smithi076 ceph-mon[63913]: from='client.54372 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:28.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:27 smithi022 ceph-mon[80039]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:28.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:27 smithi022 ceph-mon[82397]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:27 smithi076 ceph-mon[63913]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:29.022 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:31:29.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:28 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:29.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:28 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:29.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:28 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:29.658 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-09-13T23:31:30.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:29 smithi022 ceph-mon[80039]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:30.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:29 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/4136030052' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:30.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:29 smithi022 ceph-mon[82397]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:30.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:29 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/4136030052' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:29 smithi076 ceph-mon[63913]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:29 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/4136030052' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:32.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:31 smithi022 ceph-mon[80039]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:32.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:31 smithi022 ceph-mon[82397]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:32.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:31 smithi076 ceph-mon[63913]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:32.913 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:31:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:31:32] "GET /metrics HTTP/1.1" 200 38112 "" "Prometheus/2.43.0" 2024-09-13T23:31:33.013 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:31:33.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:32 smithi022 ceph-mon[80039]: from='client.54384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:33.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:32 smithi022 ceph-mon[82397]: from='client.54384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:33.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:32 smithi076 ceph-mon[63913]: from='client.54384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:33.623 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T23:31:34.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:33 smithi022 ceph-mon[80039]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:34.164 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:33 smithi022 ceph-mon[82397]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:33 smithi076 ceph-mon[63913]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:35.612 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:31:35.612 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": null, 2024-09-13T23:31:35.612 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": false, 2024-09-13T23:31:35.612 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "", 2024-09-13T23:31:35.612 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:31:35.612 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": null, 2024-09-13T23:31:35.613 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-09-13T23:31:35.613 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:31:35.613 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:31:35.916 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:35 smithi022 ceph-mon[82397]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:36.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:35 smithi022 ceph-mon[80039]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:36.259 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T23:31:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:35 smithi076 ceph-mon[63913]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:37.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:36 smithi022 ceph-mon[82397]: from='client.64324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:37.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:31:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:36 smithi022 ceph-mon[80039]: from='client.64324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:31:37.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:36 smithi076 ceph-mon[63913]: from='client.64324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:37.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:31:38.100 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:37 smithi022 ceph-mon[80039]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:38.101 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:37 smithi022 ceph-mon[82397]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:37 smithi076 ceph-mon[63913]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:38.446 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-09-13T23:31:39.131 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:38 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:39.131 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:38 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/4072598342' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:31:39.131 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:38 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:39.131 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:38 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/4072598342' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:31:39.155 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-09-13T23:31:39.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:38 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:39.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:38 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/4072598342' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:31:40.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:39 smithi022 ceph-mon[80039]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:40.137 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:39 smithi022 ceph-mon[82397]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:39 smithi076 ceph-mon[63913]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:41.952 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:41 smithi022 ceph-mon[80039]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:41.953 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:41 smithi022 ceph-mon[82397]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:41 smithi076 ceph-mon[63913]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:42.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:31:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:31:42] "GET /metrics HTTP/1.1" 200 38116 "" "Prometheus/2.43.0" 2024-09-13T23:31:42.723 INFO:teuthology.orchestra.run.smithi022.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:31:43.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:42 smithi022 ceph-mon[80039]: from='client.54402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:43.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:43.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:43.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:31:43.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:43.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:42 smithi022 ceph-mon[82397]: from='client.54402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:43.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:43.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:43.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:31:43.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:42 smithi076 ceph-mon[63913]: from='client.54402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:43.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:31:43.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:43.469 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T23:31:44.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:43 smithi022 ceph-mon[80039]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:44.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:43 smithi022 ceph-mon[80039]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:31:44.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:43 smithi022 ceph-mon[80039]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:31:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:43 smithi022 ceph-mon[82397]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:43 smithi022 ceph-mon[82397]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:31:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:43 smithi022 ceph-mon[82397]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:31:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:43 smithi076 ceph-mon[63913]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:43 smithi076 ceph-mon[63913]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:31:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:43 smithi076 ceph-mon[63913]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:31:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:45.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:31:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:31:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:31:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mgr 2024-09-13T23:31:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mon 2024-09-13T23:31:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all crash 2024-09-13T23:31:45.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:45.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T23:31:45.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T23:31:45.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[80039]: Upgrade: osd.3 is safe to restart 2024-09-13T23:31:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:31:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:31:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:31:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mgr 2024-09-13T23:31:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:45.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:45.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mon 2024-09-13T23:31:45.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:45.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:45.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all crash 2024-09-13T23:31:45.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:45.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T23:31:45.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T23:31:45.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:45 smithi022 ceph-mon[82397]: Upgrade: osd.3 is safe to restart 2024-09-13T23:31:46.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mgr 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mon 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all crash 2024-09-13T23:31:46.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:46.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T23:31:46.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T23:31:46.039 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:45 smithi076 ceph-mon[63913]: Upgrade: osd.3 is safe to restart 2024-09-13T23:31:46.915 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:31:47.633 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:31:47.633 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (18m) 40s ago 26m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:31:47.633 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (5m) 3m ago 26m 82.8M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:31:47.633 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (6m) 40s ago 25m 58.1M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (5m) 3m ago 31m 437M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (15m) 40s ago 33m 501M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (3m) 40s ago 33m 51.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (4m) 3m ago 31m 35.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (3m) 40s ago 32m 39.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (19m) 40s ago 27m 16.4M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (18m) 3m ago 27m 16.1M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (2m) 40s ago 31m 51.3M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (97s) 40s ago 30m 52.2M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (46s) 40s ago 30m 11.7M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 42db9ab27cb1 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (30m) 40s ago 30m 56.1M 2993M 17.2.0 e1d6a67b021e 0b47248ec1dc 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (29m) 3m ago 29m 56.1M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (29m) 3m ago 29m 56.0M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:31:47.634 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (28m) 3m ago 28m 55.0M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:31:47.635 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (28m) 3m ago 28m 54.5M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:31:47.635 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (5m) 3m ago 27m 60.8M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:31:47.635 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (26m) 40s ago 26m 78.2M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:31:47.635 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (26m) 3m ago 26m 76.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:31:47.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:47 smithi022 ceph-mon[80039]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:47.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:47 smithi022 ceph-mon[80039]: from='client.64342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:47.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:47.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T23:31:47.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:47.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:47 smithi022 ceph-mon[82397]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:47.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:47 smithi022 ceph-mon[82397]: from='client.64342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:47.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:47.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T23:31:47.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:48.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:47 smithi076 ceph-mon[63913]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:48.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:47 smithi076 ceph-mon[63913]: from='client.64342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:48.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:48.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T23:31:48.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:31:48.075 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:31:48.075 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:31:48.075 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:31:48.075 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:31:48.075 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:31:48.075 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:31:48.075 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:31:48.076 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:31:48.076 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T23:31:48.076 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:31:48.076 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:31:48.076 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:31:48.076 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:31:48.076 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:31:48.076 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:31:48.076 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T23:31:48.076 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T23:31:48.076 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:31:48.076 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:31:48.431 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:31:48.431 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:31:48.431 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:31:48.431 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-13T23:31:48.432 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:31:48.432 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "3/8 daemons upgraded", 2024-09-13T23:31:48.432 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T23:31:48.432 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:31:48.432 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:31:48.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:48 smithi022 ceph-mon[82397]: from='client.54414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:48.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:48 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:48.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:48 smithi022 ceph-mon[82397]: Upgrade: Updating osd.3 2024-09-13T23:31:48.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:48 smithi022 ceph-mon[82397]: Deploying daemon osd.3 on smithi022 2024-09-13T23:31:48.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:48 smithi022 ceph-mon[82397]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:48.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:48 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/2386075425' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:48.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:48 smithi022 ceph-mon[80039]: from='client.54414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:48.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:48 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:48.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:48 smithi022 ceph-mon[80039]: Upgrade: Updating osd.3 2024-09-13T23:31:48.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:48 smithi022 ceph-mon[80039]: Deploying daemon osd.3 on smithi022 2024-09-13T23:31:48.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:48 smithi022 ceph-mon[80039]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:48.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:48 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/2386075425' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:49.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:48 smithi076 ceph-mon[63913]: from='client.54414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:49.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:48 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:49.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:48 smithi076 ceph-mon[63913]: Upgrade: Updating osd.3 2024-09-13T23:31:49.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:48 smithi076 ceph-mon[63913]: Deploying daemon osd.3 on smithi022 2024-09-13T23:31:49.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:48 smithi076 ceph-mon[63913]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:49.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:48 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/2386075425' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:31:49.757 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:49 smithi022 ceph-mon[82397]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:49.757 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:49 smithi022 ceph-mon[82397]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:49.757 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:49 smithi022 ceph-mon[80039]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:49.757 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:49 smithi022 ceph-mon[80039]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:50.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:49 smithi076 ceph-mon[63913]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:31:50.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:49 smithi076 ceph-mon[63913]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:31:51.132 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:50 smithi022 systemd[1]: Stopping Ceph osd.3 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:31:51.427 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[40115]: 2024-09-13T23:31:51.129+0000 7fa03d473700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:31:51.427 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[40115]: 2024-09-13T23:31:51.129+0000 7fa03d473700 -1 osd.3 128 *** Got signal Terminated *** 2024-09-13T23:31:51.427 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:51 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[40115]: 2024-09-13T23:31:51.129+0000 7fa03d473700 -1 osd.3 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:31:51.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:51 smithi022 ceph-mon[82397]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:51.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:51 smithi022 ceph-mon[82397]: osd.3 marked itself down and dead 2024-09-13T23:31:51.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:51 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:31:51.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:51 smithi022 ceph-mon[80039]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:51.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:51 smithi022 ceph-mon[80039]: osd.3 marked itself down and dead 2024-09-13T23:31:51.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:51 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:31:52.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:51 smithi076 ceph-mon[63913]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 161 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:31:52.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:51 smithi076 ceph-mon[63913]: osd.3 marked itself down and dead 2024-09-13T23:31:52.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:51 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:31:52.427 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:52 smithi022 podman[107572]: 2024-09-13 23:31:52.194053283 +0000 UTC m=+1.214314710 container died 0b47248ec1dcdfd51abbec99312569fa2c81337b0d8478bde8a439c3b0471430 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_BRANCH=HEAD, vcs-type=git, ceph=True, GIT_CLEAN=True, io.buildah.version=1.19.8, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.expose-services=) 2024-09-13T23:31:52.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:31:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:31:52] "GET /metrics HTTP/1.1" 200 38114 "" "Prometheus/2.43.0" 2024-09-13T23:31:52.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:52 smithi022 ceph-mon[80039]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:31:52.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:52 smithi022 ceph-mon[80039]: osdmap e129: 8 total, 7 up, 8 in 2024-09-13T23:31:52.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:52 smithi022 ceph-mon[82397]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:31:52.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:52 smithi022 ceph-mon[82397]: osdmap e129: 8 total, 7 up, 8 in 2024-09-13T23:31:52.928 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:52 smithi022 podman[107572]: 2024-09-13 23:31:52.662865247 +0000 UTC m=+1.683126662 container cleanup 0b47248ec1dcdfd51abbec99312569fa2c81337b0d8478bde8a439c3b0471430 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, name=centos-stream, architecture=x86_64, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc.) 2024-09-13T23:31:52.928 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:52 smithi022 bash[107572]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3 2024-09-13T23:31:53.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:52 smithi076 ceph-mon[63913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:31:53.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:52 smithi076 ceph-mon[63913]: osdmap e129: 8 total, 7 up, 8 in 2024-09-13T23:31:53.260 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:52 smithi022 podman[107589]: 2024-09-13 23:31:52.942880143 +0000 UTC m=+0.748602373 container remove 0b47248ec1dcdfd51abbec99312569fa2c81337b0d8478bde8a439c3b0471430 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=HEAD, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.openshift.expose-services=, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, name=centos-stream, io.buildah.version=1.19.8, release=754, version=8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-13T23:31:53.560 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:53 smithi022 podman[107678]: 2024-09-13 23:31:53.162199713 +0000 UTC m=+0.016412199 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:31:53.560 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:53 smithi022 podman[107678]: 2024-09-13 23:31:53.331910637 +0000 UTC m=+0.186123105 container create 636c0fae572bb0f3b005416e1bd307bf57e4cb626f52420ad9bff4e5aa166cd3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:31:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:53 smithi022 ceph-mon[80039]: pgmap v132: 161 pgs: 45 peering, 8 stale+active+clean, 108 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:31:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:53 smithi022 ceph-mon[80039]: Health check failed: Reduced data availability: 11 pgs inactive, 18 pgs peering (PG_AVAILABILITY) 2024-09-13T23:31:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:53 smithi022 ceph-mon[80039]: osdmap e130: 8 total, 7 up, 8 in 2024-09-13T23:31:53.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:53 smithi022 ceph-mon[82397]: pgmap v132: 161 pgs: 45 peering, 8 stale+active+clean, 108 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:31:53.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:53 smithi022 ceph-mon[82397]: Health check failed: Reduced data availability: 11 pgs inactive, 18 pgs peering (PG_AVAILABILITY) 2024-09-13T23:31:53.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:53 smithi022 ceph-mon[82397]: osdmap e130: 8 total, 7 up, 8 in 2024-09-13T23:31:53.928 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:53 smithi022 podman[107678]: 2024-09-13 23:31:53.559448335 +0000 UTC m=+0.413660805 container init 636c0fae572bb0f3b005416e1bd307bf57e4cb626f52420ad9bff4e5aa166cd3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD) 2024-09-13T23:31:53.928 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:53 smithi022 podman[107678]: 2024-09-13 23:31:53.563023666 +0000 UTC m=+0.417236145 container start 636c0fae572bb0f3b005416e1bd307bf57e4cb626f52420ad9bff4e5aa166cd3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-13T23:31:53.928 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:53 smithi022 podman[107678]: 2024-09-13 23:31:53.66804979 +0000 UTC m=+0.522262262 container attach 636c0fae572bb0f3b005416e1bd307bf57e4cb626f52420ad9bff4e5aa166cd3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T23:31:53.928 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:53 smithi022 podman[107678]: 2024-09-13 23:31:53.759852317 +0000 UTC m=+0.614064800 container died 636c0fae572bb0f3b005416e1bd307bf57e4cb626f52420ad9bff4e5aa166cd3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-13T23:31:54.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:53 smithi076 ceph-mon[63913]: pgmap v132: 161 pgs: 45 peering, 8 stale+active+clean, 108 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:31:54.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:53 smithi076 ceph-mon[63913]: Health check failed: Reduced data availability: 11 pgs inactive, 18 pgs peering (PG_AVAILABILITY) 2024-09-13T23:31:54.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:53 smithi076 ceph-mon[63913]: osdmap e130: 8 total, 7 up, 8 in 2024-09-13T23:31:54.890 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:54 smithi022 podman[107701]: 2024-09-13 23:31:54.446947745 +0000 UTC m=+0.678492526 container remove 636c0fae572bb0f3b005416e1bd307bf57e4cb626f52420ad9bff4e5aa166cd3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T23:31:54.891 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:54 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T23:31:54.891 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:54 smithi022 systemd[1]: Stopped Ceph osd.3 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:31:54.891 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:54 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.3.service: Consumed 17.900s CPU time. 2024-09-13T23:31:54.891 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:54 smithi022 systemd[1]: Starting Ceph osd.3 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:31:55.176 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:54 smithi022 podman[107803]: 2024-09-13 23:31:54.889729478 +0000 UTC m=+0.099698385 container create 44e0f48a7b9ec09e3154b8927afb807a3fed555915dda9e9363a327693766c94 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:31:55.177 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:54 smithi022 podman[107803]: 2024-09-13 23:31:54.806571992 +0000 UTC m=+0.016540909 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:31:55.177 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:55 smithi022 podman[107803]: 2024-09-13 23:31:55.10046881 +0000 UTC m=+0.310437724 container init 44e0f48a7b9ec09e3154b8927afb807a3fed555915dda9e9363a327693766c94 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T23:31:55.177 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:55 smithi022 podman[107803]: 2024-09-13 23:31:55.10390708 +0000 UTC m=+0.313875994 container start 44e0f48a7b9ec09e3154b8927afb807a3fed555915dda9e9363a327693766c94 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-13T23:31:55.646 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:55 smithi022 podman[107803]: 2024-09-13 23:31:55.209610392 +0000 UTC m=+0.419579317 container attach 44e0f48a7b9ec09e3154b8927afb807a3fed555915dda9e9363a327693766c94 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:31:55.646 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate[107818]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:31:55.646 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:55 smithi022 bash[107803]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:31:55.647 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate[107818]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:31:55.647 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:55 smithi022 bash[107803]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:31:55.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:55 smithi022 ceph-mon[80039]: pgmap v134: 161 pgs: 52 peering, 6 stale+active+clean, 103 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:31:55.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:55 smithi022 ceph-mon[82397]: pgmap v134: 161 pgs: 52 peering, 6 stale+active+clean, 103 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:31:55.927 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate[107818]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T23:31:55.927 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:55 smithi022 bash[107803]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T23:31:55.927 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate[107818]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T23:31:55.927 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:55 smithi022 bash[107803]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T23:31:56.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:55 smithi076 ceph-mon[63913]: pgmap v134: 161 pgs: 52 peering, 6 stale+active+clean, 103 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:31:56.330 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate[107818]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T23:31:56.331 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 bash[107803]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T23:31:56.331 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate[107818]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T23:31:56.331 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 bash[107803]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T23:31:56.331 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate[107818]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T23:31:56.331 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 bash[107803]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T23:31:56.331 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate[107818]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T23:31:56.331 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 bash[107803]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T23:31:56.331 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate[107818]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-13T23:31:56.331 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 bash[107803]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-13T23:31:56.331 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 podman[107803]: 2024-09-13 23:31:56.035183842 +0000 UTC m=+1.245152749 container died 44e0f48a7b9ec09e3154b8927afb807a3fed555915dda9e9363a327693766c94 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-13T23:31:56.967 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:56 smithi022 ceph-mon[80039]: Health check failed: Degraded data redundancy: 22/627 objects degraded (3.509%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T23:31:56.968 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:56 smithi022 ceph-mon[82397]: Health check failed: Degraded data redundancy: 22/627 objects degraded (3.509%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T23:31:56.968 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 podman[107957]: 2024-09-13 23:31:56.754668553 +0000 UTC m=+0.710145975 container remove 44e0f48a7b9ec09e3154b8927afb807a3fed555915dda9e9363a327693766c94 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-ecab7be) 2024-09-13T23:31:57.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:56 smithi076 ceph-mon[63913]: Health check failed: Degraded data redundancy: 22/627 objects degraded (3.509%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T23:31:57.283 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 podman[108019]: 2024-09-13 23:31:56.869876874 +0000 UTC m=+0.016454922 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:31:57.284 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:56 smithi022 podman[108019]: 2024-09-13 23:31:56.972114033 +0000 UTC m=+0.118692110 container create b891533f55d351154599368044551ccaec8d701db7096344052e804c9d6a83aa (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:31:57.284 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:57 smithi022 podman[108019]: 2024-09-13 23:31:57.18275771 +0000 UTC m=+0.329335745 container init b891533f55d351154599368044551ccaec8d701db7096344052e804c9d6a83aa (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-13T23:31:57.284 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:57 smithi022 podman[108019]: 2024-09-13 23:31:57.185981012 +0000 UTC m=+0.332559049 container start b891533f55d351154599368044551ccaec8d701db7096344052e804c9d6a83aa (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-13T23:31:57.284 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:57 smithi022 ceph-osd[108037]: -- 172.21.15.22:0/3042915377 <== mon.0 v2:172.21.15.22:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a26fe17a40 con 0x55a270bde000 2024-09-13T23:31:57.676 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:57 smithi022 bash[108019]: b891533f55d351154599368044551ccaec8d701db7096344052e804c9d6a83aa 2024-09-13T23:31:57.677 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:57 smithi022 systemd[1]: Started Ceph osd.3 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:31:58.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:57 smithi076 ceph-mon[63913]: pgmap v135: 161 pgs: 9 active+undersized, 52 peering, 5 active+undersized+degraded, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 22/627 objects degraded (3.509%) 2024-09-13T23:31:58.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:58.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:58.071 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:57 smithi022 ceph-mon[80039]: pgmap v135: 161 pgs: 9 active+undersized, 52 peering, 5 active+undersized+degraded, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 22/627 objects degraded (3.509%) 2024-09-13T23:31:58.071 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:58.071 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:58.071 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:57 smithi022 ceph-mon[82397]: pgmap v135: 161 pgs: 9 active+undersized, 52 peering, 5 active+undersized+degraded, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 22/627 objects degraded (3.509%) 2024-09-13T23:31:58.071 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:58.071 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:31:58.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:58 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:58.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:58 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:58.928 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:31:58 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[108033]: 2024-09-13T23:31:58.562+0000 7f508218d640 -1 Falling back to public interface 2024-09-13T23:31:59.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:58 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:31:59.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:59 smithi022 ceph-mon[80039]: pgmap v136: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-09-13T23:31:59.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:31:59 smithi022 ceph-mon[80039]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 18 pgs peering) 2024-09-13T23:31:59.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:59 smithi022 ceph-mon[82397]: pgmap v136: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-09-13T23:31:59.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:31:59 smithi022 ceph-mon[82397]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 18 pgs peering) 2024-09-13T23:32:00.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:59 smithi076 ceph-mon[63913]: pgmap v136: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-09-13T23:32:00.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:31:59 smithi076 ceph-mon[63913]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 18 pgs peering) 2024-09-13T23:32:02.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:01 smithi076 ceph-mon[63913]: pgmap v137: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-09-13T23:32:02.093 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:01 smithi022 ceph-mon[80039]: pgmap v137: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-09-13T23:32:02.093 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:01 smithi022 ceph-mon[82397]: pgmap v137: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-09-13T23:32:02.426 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:32:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[108033]: 2024-09-13T23:32:02.091+0000 7f508218d640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-13T23:32:02.427 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:32:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[108033]: 2024-09-13T23:32:02.124+0000 7f508218d640 -1 osd.3 128 log_to_monitors true 2024-09-13T23:32:02.710 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:32:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:32:02] "GET /metrics HTTP/1.1" 200 38114 "" "Prometheus/2.43.0" 2024-09-13T23:32:03.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:02 smithi076 ceph-mon[63913]: from='osd.3 [v2:172.21.15.22:6826/2731237328,v1:172.21.15.22:6827/2731237328]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T23:32:03.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:02 smithi076 ceph-mon[63913]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T23:32:03.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:02 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:03.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:02 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:03.177 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:32:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[108033]: 2024-09-13T23:32:02.735+0000 7f5079f38640 -1 osd.3 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:32:03.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:02 smithi022 ceph-mon[80039]: from='osd.3 [v2:172.21.15.22:6826/2731237328,v1:172.21.15.22:6827/2731237328]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T23:32:03.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:02 smithi022 ceph-mon[80039]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T23:32:03.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:02 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:03.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:02 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:03.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:02 smithi022 ceph-mon[82397]: from='osd.3 [v2:172.21.15.22:6826/2731237328,v1:172.21.15.22:6827/2731237328]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T23:32:03.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:02 smithi022 ceph-mon[82397]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T23:32:03.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:02 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:03.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:02 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:03.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[80039]: pgmap v138: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s; 79/627 objects degraded (12.600%) 2024-09-13T23:32:03.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[80039]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T23:32:03.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[80039]: from='osd.3 [v2:172.21.15.22:6826/2731237328,v1:172.21.15.22:6827/2731237328]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:32:03.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[80039]: osdmap e131: 8 total, 7 up, 8 in 2024-09-13T23:32:03.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[80039]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:32:03.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:03.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:03.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[82397]: pgmap v138: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s; 79/627 objects degraded (12.600%) 2024-09-13T23:32:03.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[82397]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T23:32:03.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[82397]: from='osd.3 [v2:172.21.15.22:6826/2731237328,v1:172.21.15.22:6827/2731237328]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:32:03.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[82397]: osdmap e131: 8 total, 7 up, 8 in 2024-09-13T23:32:03.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[82397]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:32:03.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:03.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:03 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:04.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:03 smithi076 ceph-mon[63913]: pgmap v138: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 162 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s; 79/627 objects degraded (12.600%) 2024-09-13T23:32:04.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:03 smithi076 ceph-mon[63913]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T23:32:04.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:03 smithi076 ceph-mon[63913]: from='osd.3 [v2:172.21.15.22:6826/2731237328,v1:172.21.15.22:6827/2731237328]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:32:04.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:03 smithi076 ceph-mon[63913]: osdmap e131: 8 total, 7 up, 8 in 2024-09-13T23:32:04.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:03 smithi076 ceph-mon[63913]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-13T23:32:04.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:03 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:04.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:03 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:04.898 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:04 smithi022 ceph-mon[80039]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:32:04.898 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:04 smithi022 ceph-mon[80039]: osd.3 [v2:172.21.15.22:6826/2731237328,v1:172.21.15.22:6827/2731237328] boot 2024-09-13T23:32:04.898 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:04 smithi022 ceph-mon[80039]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T23:32:04.898 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:04 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:32:04.898 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:04 smithi022 ceph-mon[80039]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T23:32:04.899 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:04 smithi022 ceph-mon[82397]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:32:04.899 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:04 smithi022 ceph-mon[82397]: osd.3 [v2:172.21.15.22:6826/2731237328,v1:172.21.15.22:6827/2731237328] boot 2024-09-13T23:32:04.899 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:04 smithi022 ceph-mon[82397]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T23:32:04.899 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:04 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:32:04.899 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:04 smithi022 ceph-mon[82397]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T23:32:05.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:04 smithi076 ceph-mon[63913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:32:05.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:04 smithi076 ceph-mon[63913]: osd.3 [v2:172.21.15.22:6826/2731237328,v1:172.21.15.22:6827/2731237328] boot 2024-09-13T23:32:05.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:04 smithi076 ceph-mon[63913]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T23:32:05.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:04 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T23:32:05.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:04 smithi076 ceph-mon[63913]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T23:32:05.885 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:05 smithi022 ceph-mon[80039]: pgmap v141: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 79/627 objects degraded (12.600%) 2024-09-13T23:32:05.886 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:05 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 79/627 objects degraded (12.600%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T23:32:05.886 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:05 smithi022 ceph-mon[82397]: pgmap v141: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 79/627 objects degraded (12.600%) 2024-09-13T23:32:05.886 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:05 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 79/627 objects degraded (12.600%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T23:32:06.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:05 smithi076 ceph-mon[63913]: pgmap v141: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 79/627 objects degraded (12.600%) 2024-09-13T23:32:06.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:05 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 79/627 objects degraded (12.600%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T23:32:06.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:32:06 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:32:06.408+0000 7fd7d6194640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-13T23:32:07.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:32:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:32:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:32:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:32:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:07.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T23:32:07.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:07.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:07.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:32:07.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:32:07.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:07.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:07.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:32:07.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:32:07.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:07.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:07.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:07.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T23:32:07.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:07.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:07.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:32:07.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:32:07.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:07.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:07.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:32:07.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:32:07.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:07.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:07.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:07.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T23:32:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:08 smithi076 ceph-mon[63913]: pgmap v143: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 65/627 objects degraded (10.367%) 2024-09-13T23:32:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:08 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T23:32:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:08 smithi076 ceph-mon[63913]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-13T23:32:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:08 smithi022 ceph-mon[80039]: pgmap v143: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 65/627 objects degraded (10.367%) 2024-09-13T23:32:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:08 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T23:32:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:08 smithi022 ceph-mon[80039]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-13T23:32:08.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:08 smithi022 ceph-mon[82397]: pgmap v143: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 65/627 objects degraded (10.367%) 2024-09-13T23:32:08.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:08 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T23:32:08.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:08 smithi022 ceph-mon[82397]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-13T23:32:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:09 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:09.578 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:09 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:09.578 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:09 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:10.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:10 smithi076 ceph-mon[63913]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:10.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:10 smithi076 ceph-mon[63913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 65/627 objects degraded (10.367%), 21 pgs degraded) 2024-09-13T23:32:10.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:10 smithi076 ceph-mon[63913]: Cluster is now healthy 2024-09-13T23:32:10.590 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:10 smithi022 ceph-mon[80039]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:10.590 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:10 smithi022 ceph-mon[80039]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 65/627 objects degraded (10.367%), 21 pgs degraded) 2024-09-13T23:32:10.590 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:10 smithi022 ceph-mon[80039]: Cluster is now healthy 2024-09-13T23:32:10.591 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:10 smithi022 ceph-mon[82397]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:10.591 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:10 smithi022 ceph-mon[82397]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 65/627 objects degraded (10.367%), 21 pgs degraded) 2024-09-13T23:32:10.591 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:10 smithi022 ceph-mon[82397]: Cluster is now healthy 2024-09-13T23:32:11.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:11 smithi022 ceph-mon[80039]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:11.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:11 smithi022 ceph-mon[82397]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:11.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:11 smithi076 ceph-mon[63913]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:12.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:12 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:12.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:32:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:32:12] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.43.0" 2024-09-13T23:32:12.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:12 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:12.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:12 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:13.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:13 smithi022 ceph-mon[80039]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:13.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:13 smithi022 ceph-mon[82397]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:13.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:13 smithi076 ceph-mon[63913]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:15.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:15 smithi022 ceph-mon[80039]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:15.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:15 smithi022 ceph-mon[82397]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:15.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:15 smithi076 ceph-mon[63913]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:17.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:17 smithi022 ceph-mon[80039]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:17.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:17 smithi022 ceph-mon[82397]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:17.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:17 smithi076 ceph-mon[63913]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:32:18.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:18 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:18.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:18 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:18.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:18 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:18.805 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:32:19.489 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:32:19.489 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (19m) 17s ago 27m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:32:19.489 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (6m) 4m ago 27m 82.8M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:32:19.489 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (6m) 17s ago 26m 58.7M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:32:19.489 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (6m) 4m ago 32m 437M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:32:19.489 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (16m) 17s ago 33m 503M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:32:19.489 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (4m) 17s ago 33m 53.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:32:19.489 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (5m) 4m ago 32m 35.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:32:19.489 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (3m) 17s ago 32m 42.5M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:32:19.489 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (19m) 17s ago 28m 16.1M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:32:19.489 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (19m) 4m ago 28m 16.1M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:32:19.490 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (2m) 17s ago 31m 51.8M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:32:19.490 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (2m) 17s ago 31m 52.8M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:32:19.490 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (78s) 17s ago 30m 48.6M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 42db9ab27cb1 2024-09-13T23:32:19.490 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (22s) 17s ago 30m 14.5M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 b891533f55d3 2024-09-13T23:32:19.490 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (30m) 4m ago 30m 56.1M 3248M 17.2.0 e1d6a67b021e 02fe8e14bef6 2024-09-13T23:32:19.490 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (29m) 4m ago 29m 56.0M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:32:19.490 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (29m) 4m ago 29m 55.0M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:32:19.490 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (29m) 4m ago 29m 54.5M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:32:19.490 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (6m) 4m ago 28m 60.8M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:32:19.490 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (26m) 17s ago 26m 78.5M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:32:19.490 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (26m) 4m ago 26m 76.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:32:19.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:19 smithi022 ceph-mon[80039]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T23:32:19.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:19 smithi022 ceph-mon[80039]: from='client.54444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:19.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:19 smithi022 ceph-mon[80039]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:19 smithi022 ceph-mon[82397]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T23:32:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:19 smithi022 ceph-mon[82397]: from='client.54444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:19 smithi022 ceph-mon[82397]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:19.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:19 smithi076 ceph-mon[63913]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T23:32:19.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:19 smithi076 ceph-mon[63913]: from='client.54444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:19.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:19 smithi076 ceph-mon[63913]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:19.906 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:32:19.907 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:32:19.907 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:32:19.907 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:32:19.907 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:32:19.907 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:32:19.907 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:32:19.908 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:32:19.908 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-13T23:32:19.908 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 4 2024-09-13T23:32:19.908 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:32:19.908 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:32:19.908 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:32:19.909 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:32:19.909 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:32:19.909 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-13T23:32:19.909 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 9 2024-09-13T23:32:19.909 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:32:19.909 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:32:20.256 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:32:20.256 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:32:20.256 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:32:20.256 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-13T23:32:20.256 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:32:20.256 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "4/8 daemons upgraded", 2024-09-13T23:32:20.256 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T23:32:20.256 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:32:20.256 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:32:20.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:20 smithi022 ceph-mon[82397]: from='client.54456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:20.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:20 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/817570805' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:20.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:20 smithi022 ceph-mon[82397]: from='client.54468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:20 smithi022 ceph-mon[80039]: from='client.54456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:20 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/817570805' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:20 smithi022 ceph-mon[80039]: from='client.54468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:20.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:20 smithi076 ceph-mon[63913]: from='client.54456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:20.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:20 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/817570805' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:20.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:20 smithi076 ceph-mon[63913]: from='client.54468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:21.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:21 smithi022 ceph-mon[80039]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:32:21.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:21.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:32:21.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:21 smithi022 ceph-mon[82397]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:32:21.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:21.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:32:21.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:21 smithi076 ceph-mon[63913]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:32:21.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:21.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:32:22.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:32:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:32:22] "GET /metrics HTTP/1.1" 200 38125 "" "Prometheus/2.43.0" 2024-09-13T23:32:22.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T23:32:22.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:22 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T23:32:22.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:22 smithi022 ceph-mon[80039]: Upgrade: osd.4 is safe to restart 2024-09-13T23:32:22.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T23:32:22.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:22 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T23:32:22.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:22 smithi022 ceph-mon[82397]: Upgrade: osd.4 is safe to restart 2024-09-13T23:32:22.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T23:32:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:22 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T23:32:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:22 smithi076 ceph-mon[63913]: Upgrade: osd.4 is safe to restart 2024-09-13T23:32:24.502 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:24 smithi076 ceph-mon[63913]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:32:24.502 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:24 smithi076 ceph-mon[63913]: Upgrade: Updating osd.4 2024-09-13T23:32:24.502 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:24 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:24.502 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:24 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T23:32:24.502 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:24 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:32:24.502 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:24 smithi076 ceph-mon[63913]: Deploying daemon osd.4 on smithi076 2024-09-13T23:32:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:24 smithi022 ceph-mon[80039]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:32:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:24 smithi022 ceph-mon[80039]: Upgrade: Updating osd.4 2024-09-13T23:32:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:24 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:24 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T23:32:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:24 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:32:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:24 smithi022 ceph-mon[80039]: Deploying daemon osd.4 on smithi076 2024-09-13T23:32:24.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:24 smithi022 ceph-mon[82397]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:32:24.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:24 smithi022 ceph-mon[82397]: Upgrade: Updating osd.4 2024-09-13T23:32:24.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:24 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:24.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:24 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T23:32:24.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:24 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:32:24.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:24 smithi022 ceph-mon[82397]: Deploying daemon osd.4 on smithi076 2024-09-13T23:32:26.416 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:26 smithi076 ceph-mon[63913]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:32:26.416 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:26 smithi076 systemd[1]: Stopping Ceph osd.4 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:32:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:26 smithi022 ceph-mon[80039]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:32:26.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:26 smithi022 ceph-mon[82397]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:32:26.787 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:26 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[27943]: 2024-09-13T23:32:26.414+0000 7f1b1967a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:32:26.787 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:26 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[27943]: 2024-09-13T23:32:26.414+0000 7f1b1967a700 -1 osd.4 133 *** Got signal Terminated *** 2024-09-13T23:32:26.787 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:26 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[27943]: 2024-09-13T23:32:26.414+0000 7f1b1967a700 -1 osd.4 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:32:27.483 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:27 smithi022 ceph-mon[80039]: osd.4 marked itself down and dead 2024-09-13T23:32:27.484 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:27 smithi022 ceph-mon[82397]: osd.4 marked itself down and dead 2024-09-13T23:32:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:27 smithi076 ceph-mon[63913]: osd.4 marked itself down and dead 2024-09-13T23:32:27.976 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:27 smithi076 podman[74039]: 2024-09-13 23:32:27.710285525 +0000 UTC m=+1.407717258 container died 02fe8e14bef69f7a334fa42d03e0ed91251eaed5a43c5c281487474fbe33c88f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4, GIT_CLEAN=True, release=754, version=8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, vcs-type=git, RELEASE=HEAD) 2024-09-13T23:32:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:28 smithi076 ceph-mon[63913]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:32:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:28 smithi076 ceph-mon[63913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:32:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:28 smithi076 ceph-mon[63913]: osdmap e134: 8 total, 7 up, 8 in 2024-09-13T23:32:28.288 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:28 smithi076 podman[74039]: 2024-09-13 23:32:28.081819237 +0000 UTC m=+1.779250962 container cleanup 02fe8e14bef69f7a334fa42d03e0ed91251eaed5a43c5c281487474fbe33c88f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4, io.openshift.expose-services=, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, GIT_CLEAN=True, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vcs-type=git, io.buildah.version=1.19.8, release=754, name=centos-stream) 2024-09-13T23:32:28.288 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:28 smithi076 bash[74039]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4 2024-09-13T23:32:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:28 smithi022 ceph-mon[80039]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:32:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:28 smithi022 ceph-mon[80039]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:32:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:28 smithi022 ceph-mon[80039]: osdmap e134: 8 total, 7 up, 8 in 2024-09-13T23:32:28.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:28 smithi022 ceph-mon[82397]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:32:28.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:28 smithi022 ceph-mon[82397]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:32:28.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:28 smithi022 ceph-mon[82397]: osdmap e134: 8 total, 7 up, 8 in 2024-09-13T23:32:28.685 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:28 smithi076 podman[74056]: 2024-09-13 23:32:28.383827632 +0000 UTC m=+0.674947114 container remove 02fe8e14bef69f7a334fa42d03e0ed91251eaed5a43c5c281487474fbe33c88f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, version=8, io.k8s.display-name=CentOS Stream 8, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.openshift.expose-services=, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, ceph=True, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream) 2024-09-13T23:32:28.938 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:28 smithi076 podman[74149]: 2024-09-13 23:32:28.684772914 +0000 UTC m=+0.102473072 container create 5a23da70b919dc496586dc56f3c31a9c354705de439a31a2063533baf080ac6e (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=squid-ecab7be) 2024-09-13T23:32:28.939 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:28 smithi076 podman[74149]: 2024-09-13 23:32:28.59821831 +0000 UTC m=+0.015918432 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:32:28.939 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:28 smithi076 podman[74149]: 2024-09-13 23:32:28.937830354 +0000 UTC m=+0.355530477 container init 5a23da70b919dc496586dc56f3c31a9c354705de439a31a2063533baf080ac6e (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-13T23:32:29.216 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:28 smithi076 podman[74149]: 2024-09-13 23:32:28.941304542 +0000 UTC m=+0.359004663 container start 5a23da70b919dc496586dc56f3c31a9c354705de439a31a2063533baf080ac6e (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T23:32:29.216 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:29 smithi076 podman[74149]: 2024-09-13 23:32:29.021224786 +0000 UTC m=+0.438924915 container attach 5a23da70b919dc496586dc56f3c31a9c354705de439a31a2063533baf080ac6e (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-13T23:32:29.216 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:29 smithi076 podman[74149]: 2024-09-13 23:32:29.140895691 +0000 UTC m=+0.558595808 container died 5a23da70b919dc496586dc56f3c31a9c354705de439a31a2063533baf080ac6e (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T23:32:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:29 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:29 smithi076 ceph-mon[63913]: osdmap e135: 8 total, 7 up, 8 in 2024-09-13T23:32:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:29 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:29.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:29 smithi022 ceph-mon[80039]: osdmap e135: 8 total, 7 up, 8 in 2024-09-13T23:32:29.687 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:29 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:29.687 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:29 smithi022 ceph-mon[82397]: osdmap e135: 8 total, 7 up, 8 in 2024-09-13T23:32:30.037 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:29 smithi076 podman[74171]: 2024-09-13 23:32:29.734198621 +0000 UTC m=+0.587449941 container remove 5a23da70b919dc496586dc56f3c31a9c354705de439a31a2063533baf080ac6e (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-13T23:32:30.038 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:29 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-13T23:32:30.038 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:29 smithi076 systemd[1]: Stopped Ceph osd.4 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:32:30.038 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:29 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.4.service: Consumed 16.056s CPU time. 2024-09-13T23:32:30.038 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:29 smithi076 systemd[1]: Starting Ceph osd.4 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:32:30.438 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:30 smithi076 ceph-mon[63913]: pgmap v156: 161 pgs: 49 peering, 112 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T23:32:30.438 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:30 smithi076 ceph-mon[63913]: Health check failed: Reduced data availability: 8 pgs inactive, 13 pgs peering (PG_AVAILABILITY) 2024-09-13T23:32:30.439 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:30 smithi076 podman[74268]: 2024-09-13 23:32:30.061705229 +0000 UTC m=+0.016082517 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:32:30.439 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:30 smithi076 podman[74268]: 2024-09-13 23:32:30.185114211 +0000 UTC m=+0.139491498 container create 49ee749142c34ef64884d4d8c639f6e316c79434d26c35440dbb95aa3cfe51d6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T23:32:30.439 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:30 smithi076 podman[74268]: 2024-09-13 23:32:30.438029365 +0000 UTC m=+0.392406653 container init 49ee749142c34ef64884d4d8c639f6e316c79434d26c35440dbb95aa3cfe51d6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2) 2024-09-13T23:32:30.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:30 smithi022 ceph-mon[82397]: pgmap v156: 161 pgs: 49 peering, 112 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T23:32:30.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:30 smithi022 ceph-mon[82397]: Health check failed: Reduced data availability: 8 pgs inactive, 13 pgs peering (PG_AVAILABILITY) 2024-09-13T23:32:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:30 smithi022 ceph-mon[80039]: pgmap v156: 161 pgs: 49 peering, 112 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T23:32:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:30 smithi022 ceph-mon[80039]: Health check failed: Reduced data availability: 8 pgs inactive, 13 pgs peering (PG_AVAILABILITY) 2024-09-13T23:32:30.787 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:30 smithi076 podman[74268]: 2024-09-13 23:32:30.441587256 +0000 UTC m=+0.395964552 container start 49ee749142c34ef64884d4d8c639f6e316c79434d26c35440dbb95aa3cfe51d6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T23:32:30.787 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:30 smithi076 podman[74268]: 2024-09-13 23:32:30.52136532 +0000 UTC m=+0.475742610 container attach 49ee749142c34ef64884d4d8c639f6e316c79434d26c35440dbb95aa3cfe51d6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T23:32:30.788 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate[74283]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:32:30.788 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:30 smithi076 bash[74268]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:32:30.788 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate[74283]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:32:30.788 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:30 smithi076 bash[74268]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:32:32.219 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate[74283]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T23:32:32.220 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:31 smithi076 bash[74268]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T23:32:32.220 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate[74283]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T23:32:32.220 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:31 smithi076 bash[74268]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T23:32:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:32 smithi076 ceph-mon[63913]: pgmap v157: 161 pgs: 49 peering, 112 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T23:32:32.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate[74283]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T23:32:32.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:32 smithi076 bash[74268]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T23:32:32.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate[74283]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T23:32:32.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:32 smithi076 bash[74268]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T23:32:32.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate[74283]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T23:32:32.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:32 smithi076 bash[74268]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T23:32:32.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate[74283]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T23:32:32.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:32 smithi076 bash[74268]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T23:32:32.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate[74283]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-13T23:32:32.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:32 smithi076 bash[74268]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-13T23:32:32.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:32 smithi076 podman[74268]: 2024-09-13 23:32:32.268757598 +0000 UTC m=+2.223134892 container died 49ee749142c34ef64884d4d8c639f6e316c79434d26c35440dbb95aa3cfe51d6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True) 2024-09-13T23:32:32.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:32:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:32:32] "GET /metrics HTTP/1.1" 200 38125 "" "Prometheus/2.43.0" 2024-09-13T23:32:32.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:32 smithi022 ceph-mon[82397]: pgmap v157: 161 pgs: 49 peering, 112 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T23:32:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:32 smithi022 ceph-mon[80039]: pgmap v157: 161 pgs: 49 peering, 112 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T23:32:33.175 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:32 smithi076 podman[74413]: 2024-09-13 23:32:32.859052689 +0000 UTC m=+0.584420508 container remove 49ee749142c34ef64884d4d8c639f6e316c79434d26c35440dbb95aa3cfe51d6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-13T23:32:33.175 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:33 smithi076 podman[74473]: 2024-09-13 23:32:33.060187174 +0000 UTC m=+0.106804374 container create ba143ef88b4e45c01d25623cdacea94f60ff7fbdcde9c7723a629b869343f84d (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-13T23:32:33.175 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:33 smithi076 podman[74473]: 2024-09-13 23:32:32.969413607 +0000 UTC m=+0.016030806 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:32:33.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:33 smithi076 ceph-mon[63913]: pgmap v158: 161 pgs: 27 active+undersized, 17 peering, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 96/627 objects degraded (15.311%) 2024-09-13T23:32:33.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:33 smithi076 ceph-mon[63913]: Health check failed: Degraded data redundancy: 96/627 objects degraded (15.311%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T23:32:33.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:33 smithi076 podman[74473]: 2024-09-13 23:32:33.27952223 +0000 UTC m=+0.326139424 container init ba143ef88b4e45c01d25623cdacea94f60ff7fbdcde9c7723a629b869343f84d (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:32:33.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:33 smithi076 podman[74473]: 2024-09-13 23:32:33.282818882 +0000 UTC m=+0.329436076 container start ba143ef88b4e45c01d25623cdacea94f60ff7fbdcde9c7723a629b869343f84d (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True) 2024-09-13T23:32:33.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:33 smithi076 ceph-osd[74492]: -- 172.21.15.76:0/1040275060 <== mon.2 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5591e01b9a40 con 0x5591e0f80000 2024-09-13T23:32:33.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:33 smithi076 bash[74473]: ba143ef88b4e45c01d25623cdacea94f60ff7fbdcde9c7723a629b869343f84d 2024-09-13T23:32:33.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:33 smithi076 systemd[1]: Started Ceph osd.4 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:32:33.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:33 smithi022 ceph-mon[80039]: pgmap v158: 161 pgs: 27 active+undersized, 17 peering, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 96/627 objects degraded (15.311%) 2024-09-13T23:32:33.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:33 smithi022 ceph-mon[80039]: Health check failed: Degraded data redundancy: 96/627 objects degraded (15.311%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T23:32:33.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:33 smithi022 ceph-mon[82397]: pgmap v158: 161 pgs: 27 active+undersized, 17 peering, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 96/627 objects degraded (15.311%) 2024-09-13T23:32:33.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:33 smithi022 ceph-mon[82397]: Health check failed: Degraded data redundancy: 96/627 objects degraded (15.311%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T23:32:34.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:34 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:34.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:34 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:34.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:34 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:34.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:34 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:34.771 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:34 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:34.771 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:34 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:35.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:35 smithi076 ceph-mon[63913]: pgmap v159: 161 pgs: 40 active+undersized, 28 active+undersized+degraded, 93 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-09-13T23:32:35.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:35 smithi076 ceph-mon[63913]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-09-13T23:32:35.537 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:32:35.163+0000 7f2135ebf640 -1 Falling back to public interface 2024-09-13T23:32:35.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:35 smithi022 ceph-mon[80039]: pgmap v159: 161 pgs: 40 active+undersized, 28 active+undersized+degraded, 93 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-09-13T23:32:35.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:35 smithi022 ceph-mon[80039]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-09-13T23:32:35.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:35 smithi022 ceph-mon[82397]: pgmap v159: 161 pgs: 40 active+undersized, 28 active+undersized+degraded, 93 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-09-13T23:32:35.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:35 smithi022 ceph-mon[82397]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-09-13T23:32:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:32:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:37 smithi076 ceph-mon[63913]: pgmap v160: 161 pgs: 40 active+undersized, 28 active+undersized+degraded, 93 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-09-13T23:32:37.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:37.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:32:37.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:37 smithi022 ceph-mon[80039]: pgmap v160: 161 pgs: 40 active+undersized, 28 active+undersized+degraded, 93 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-09-13T23:32:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:32:37.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:37 smithi022 ceph-mon[82397]: pgmap v160: 161 pgs: 40 active+undersized, 28 active+undersized+degraded, 93 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-09-13T23:32:38.744 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:32:38.448+0000 7f2135ebf640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-13T23:32:38.744 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:32:38.476+0000 7f2135ebf640 -1 osd.4 133 log_to_monitors true 2024-09-13T23:32:38.744 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:38 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:38.744 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:38.744 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:38.744 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:38.744 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:38.744 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:38 smithi076 ceph-mon[63913]: from='osd.4 [v2:172.21.15.76:6800/572684220,v1:172.21.15.76:6801/572684220]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T23:32:38.744 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:38 smithi076 ceph-mon[63913]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T23:32:38.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:38.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:38.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:38.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:38.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:38.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[82397]: from='osd.4 [v2:172.21.15.76:6800/572684220,v1:172.21.15.76:6801/572684220]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T23:32:38.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[82397]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T23:32:38.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:38.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:38.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:38.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:38.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:38.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[80039]: from='osd.4 [v2:172.21.15.76:6800/572684220,v1:172.21.15.76:6801/572684220]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T23:32:38.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:38 smithi022 ceph-mon[80039]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T23:32:39.989 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:32:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:32:39.792+0000 7f2129c62640 -1 osd.4 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:32:39.989 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:39 smithi076 ceph-mon[63913]: pgmap v161: 161 pgs: 40 active+undersized, 28 active+undersized+degraded, 93 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 405 B/s rd, 0 op/s; 107/627 objects degraded (17.065%) 2024-09-13T23:32:39.989 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:39 smithi076 ceph-mon[63913]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T23:32:39.990 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:39 smithi076 ceph-mon[63913]: osdmap e136: 8 total, 7 up, 8 in 2024-09-13T23:32:39.990 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:39 smithi076 ceph-mon[63913]: from='osd.4 [v2:172.21.15.76:6800/572684220,v1:172.21.15.76:6801/572684220]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:32:39.990 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:39 smithi076 ceph-mon[63913]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:32:40.088 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:39 smithi022 ceph-mon[80039]: pgmap v161: 161 pgs: 40 active+undersized, 28 active+undersized+degraded, 93 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 405 B/s rd, 0 op/s; 107/627 objects degraded (17.065%) 2024-09-13T23:32:40.088 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:39 smithi022 ceph-mon[80039]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T23:32:40.088 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:39 smithi022 ceph-mon[80039]: osdmap e136: 8 total, 7 up, 8 in 2024-09-13T23:32:40.088 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:39 smithi022 ceph-mon[80039]: from='osd.4 [v2:172.21.15.76:6800/572684220,v1:172.21.15.76:6801/572684220]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:32:40.088 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:39 smithi022 ceph-mon[80039]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:32:40.089 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:39 smithi022 ceph-mon[82397]: pgmap v161: 161 pgs: 40 active+undersized, 28 active+undersized+degraded, 93 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 405 B/s rd, 0 op/s; 107/627 objects degraded (17.065%) 2024-09-13T23:32:40.089 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:39 smithi022 ceph-mon[82397]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T23:32:40.089 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:39 smithi022 ceph-mon[82397]: osdmap e136: 8 total, 7 up, 8 in 2024-09-13T23:32:40.089 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:39 smithi022 ceph-mon[82397]: from='osd.4 [v2:172.21.15.76:6800/572684220,v1:172.21.15.76:6801/572684220]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:32:40.089 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:39 smithi022 ceph-mon[82397]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:32:41.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:40 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 107/627 objects degraded (17.065%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T23:32:41.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:40 smithi076 ceph-mon[63913]: from='osd.4 ' entity='osd.4' 2024-09-13T23:32:41.078 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:40 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 107/627 objects degraded (17.065%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T23:32:41.079 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:40 smithi022 ceph-mon[82397]: from='osd.4 ' entity='osd.4' 2024-09-13T23:32:41.079 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:40 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 107/627 objects degraded (17.065%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T23:32:41.079 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:40 smithi022 ceph-mon[80039]: from='osd.4 ' entity='osd.4' 2024-09-13T23:32:42.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:41 smithi076 ceph-mon[63913]: pgmap v163: 161 pgs: 40 active+undersized, 28 active+undersized+degraded, 93 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 107/627 objects degraded (17.065%) 2024-09-13T23:32:42.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:41 smithi076 ceph-mon[63913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:32:42.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:41 smithi076 ceph-mon[63913]: osd.4 [v2:172.21.15.76:6800/572684220,v1:172.21.15.76:6801/572684220] boot 2024-09-13T23:32:42.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:41 smithi076 ceph-mon[63913]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T23:32:42.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:32:42.177 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:32:41 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:32:41.950+0000 7fd7d6194640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-13T23:32:42.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:41 smithi022 ceph-mon[80039]: pgmap v163: 161 pgs: 40 active+undersized, 28 active+undersized+degraded, 93 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 107/627 objects degraded (17.065%) 2024-09-13T23:32:42.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:41 smithi022 ceph-mon[80039]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:32:42.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:41 smithi022 ceph-mon[80039]: osd.4 [v2:172.21.15.76:6800/572684220,v1:172.21.15.76:6801/572684220] boot 2024-09-13T23:32:42.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:41 smithi022 ceph-mon[80039]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T23:32:42.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:32:42.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:41 smithi022 ceph-mon[82397]: pgmap v163: 161 pgs: 40 active+undersized, 28 active+undersized+degraded, 93 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 107/627 objects degraded (17.065%) 2024-09-13T23:32:42.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:41 smithi022 ceph-mon[82397]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:32:42.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:41 smithi022 ceph-mon[82397]: osd.4 [v2:172.21.15.76:6800/572684220,v1:172.21.15.76:6801/572684220] boot 2024-09-13T23:32:42.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:41 smithi022 ceph-mon[82397]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T23:32:42.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T23:32:42.792 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:32:42] "GET /metrics HTTP/1.1" 200 38124 "" "Prometheus/2.43.0" 2024-09-13T23:32:43.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: osdmap e138: 8 total, 8 up, 8 in 2024-09-13T23:32:43.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:43.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:43.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:32:43.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:32:43.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:43.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:32:43.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:43.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:43.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:43.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T23:32:43.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T23:32:43.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[82397]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-13T23:32:43.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: osdmap e138: 8 total, 8 up, 8 in 2024-09-13T23:32:43.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:43.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:43.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:32:43.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:32:43.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:43.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:32:43.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:43.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:43.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:43.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T23:32:43.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T23:32:43.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:42 smithi022 ceph-mon[80039]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-13T23:32:43.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: osdmap e138: 8 total, 8 up, 8 in 2024-09-13T23:32:43.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:43.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:43.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:32:43.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:32:43.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:43.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:32:43.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:43.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:43.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:43.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T23:32:43.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T23:32:43.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:42 smithi076 ceph-mon[63913]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-13T23:32:44.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:43 smithi022 ceph-mon[80039]: pgmap v166: 161 pgs: 7 activating, 24 active+undersized, 14 active+undersized+degraded, 116 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 47/627 objects degraded (7.496%) 2024-09-13T23:32:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:43 smithi022 ceph-mon[82397]: pgmap v166: 161 pgs: 7 activating, 24 active+undersized, 14 active+undersized+degraded, 116 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 47/627 objects degraded (7.496%) 2024-09-13T23:32:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:43 smithi076 ceph-mon[63913]: pgmap v166: 161 pgs: 7 activating, 24 active+undersized, 14 active+undersized+degraded, 116 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 47/627 objects degraded (7.496%) 2024-09-13T23:32:45.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:44 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 47/627 objects degraded (7.496%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T23:32:45.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:44 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 47/627 objects degraded (7.496%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T23:32:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:44 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 47/627 objects degraded (7.496%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T23:32:46.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:45 smithi022 ceph-mon[80039]: pgmap v167: 161 pgs: 7 activating, 17 active+undersized, 13 active+undersized+degraded, 124 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 43/627 objects degraded (6.858%) 2024-09-13T23:32:46.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:45 smithi022 ceph-mon[82397]: pgmap v167: 161 pgs: 7 activating, 17 active+undersized, 13 active+undersized+degraded, 124 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 43/627 objects degraded (6.858%) 2024-09-13T23:32:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:45 smithi076 ceph-mon[63913]: pgmap v167: 161 pgs: 7 activating, 17 active+undersized, 13 active+undersized+degraded, 124 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 43/627 objects degraded (6.858%) 2024-09-13T23:32:47.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:46 smithi022 ceph-mon[80039]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/627 objects degraded (6.858%), 13 pgs degraded) 2024-09-13T23:32:47.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:46 smithi022 ceph-mon[80039]: Cluster is now healthy 2024-09-13T23:32:47.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:46 smithi022 ceph-mon[82397]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/627 objects degraded (6.858%), 13 pgs degraded) 2024-09-13T23:32:47.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:46 smithi022 ceph-mon[82397]: Cluster is now healthy 2024-09-13T23:32:47.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:46 smithi076 ceph-mon[63913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/627 objects degraded (6.858%), 13 pgs degraded) 2024-09-13T23:32:47.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:46 smithi076 ceph-mon[63913]: Cluster is now healthy 2024-09-13T23:32:48.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:47 smithi022 ceph-mon[82397]: pgmap v168: 161 pgs: 7 activating, 154 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-09-13T23:32:48.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:47 smithi022 ceph-mon[80039]: pgmap v168: 161 pgs: 7 activating, 154 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-09-13T23:32:48.259 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:47 smithi076 ceph-mon[63913]: pgmap v168: 161 pgs: 7 activating, 154 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-09-13T23:32:49.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:48 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:49.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:48 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:49.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:48 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:49 smithi022 ceph-mon[80039]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:32:50.232 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:49 smithi022 ceph-mon[82397]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:32:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:49 smithi076 ceph-mon[63913]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:32:50.635 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:32:51.330 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:32:51.330 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (19m) 49s ago 27m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (6m) 13s ago 27m 82.2M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (7m) 49s ago 27m 58.7M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (6m) 13s ago 32m 441M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (16m) 49s ago 34m 503M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (4m) 49s ago 34m 53.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (5m) 13s ago 32m 45.7M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (4m) 49s ago 33m 42.5M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (20m) 49s ago 28m 16.1M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (19m) 13s ago 28m 15.7M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (3m) 49s ago 32m 51.8M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (2m) 49s ago 31m 52.8M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (109s) 49s ago 31m 48.6M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 42db9ab27cb1 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (54s) 49s ago 31m 14.5M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 b891533f55d3 2024-09-13T23:32:51.331 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (18s) 13s ago 30m 12.1M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 ba143ef88b4e 2024-09-13T23:32:51.332 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (30m) 13s ago 30m 59.2M 3248M 17.2.0 e1d6a67b021e 0e0c75069a99 2024-09-13T23:32:51.332 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (29m) 13s ago 29m 57.8M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:32:51.332 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (29m) 13s ago 29m 59.4M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:32:51.332 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (7m) 13s ago 28m 69.4M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:32:51.332 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (27m) 49s ago 27m 78.5M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:32:51.332 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (27m) 13s ago 27m 77.5M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:32:51.752 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:32:51.753 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:32:51.753 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:32:51.753 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:32:51.753 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:32:51.753 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:32:51.753 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:32:51.753 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:32:51.753 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-13T23:32:51.753 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 5 2024-09-13T23:32:51.753 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:32:51.753 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:32:51.753 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:32:51.754 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:32:51.754 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:32:51.754 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T23:32:51.754 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 10 2024-09-13T23:32:51.754 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:32:51.754 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:32:52.104 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:32:52.105 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:32:52.105 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:32:52.105 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-13T23:32:52.105 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:32:52.105 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "5/8 daemons upgraded", 2024-09-13T23:32:52.105 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T23:32:52.105 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:32:52.105 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:32:52.105 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:51 smithi022 ceph-mon[80039]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T23:32:52.106 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:51 smithi022 ceph-mon[80039]: from='client.54477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:52.106 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:51 smithi022 ceph-mon[80039]: from='client.64417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:52.106 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:51 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:52.106 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:51 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:32:52.106 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:51 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/4003391555' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:52.106 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:51 smithi022 ceph-mon[82397]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T23:32:52.106 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:51 smithi022 ceph-mon[82397]: from='client.54477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:52.106 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:51 smithi022 ceph-mon[82397]: from='client.64417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:52.106 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:51 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:52.106 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:51 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:32:52.106 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:51 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/4003391555' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:51 smithi076 ceph-mon[63913]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T23:32:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:51 smithi076 ceph-mon[63913]: from='client.54477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:51 smithi076 ceph-mon[63913]: from='client.64417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:51 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:52.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:51 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:32:52.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:51 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/4003391555' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:32:52.903 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:32:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:32:52] "GET /metrics HTTP/1.1" 200 38133 "" "Prometheus/2.43.0" 2024-09-13T23:32:53.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:52 smithi022 ceph-mon[80039]: from='client.54489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:53.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:52 smithi022 ceph-mon[80039]: from='client.54501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:52 smithi022 ceph-mon[82397]: from='client.54489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:52 smithi022 ceph-mon[82397]: from='client.54501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:52 smithi076 ceph-mon[63913]: from='client.54489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:52 smithi076 ceph-mon[63913]: from='client.54501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:32:54.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:53 smithi022 ceph-mon[80039]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 0 op/s 2024-09-13T23:32:54.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:53 smithi022 ceph-mon[82397]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 0 op/s 2024-09-13T23:32:54.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:53 smithi076 ceph-mon[63913]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 0 op/s 2024-09-13T23:32:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:55 smithi022 ceph-mon[80039]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:32:56.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:55 smithi022 ceph-mon[82397]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:32:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:55 smithi076 ceph-mon[63913]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:32:57.654 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:57 smithi076 ceph-mon[63913]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:32:57.654 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:57.654 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T23:32:57.654 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:57 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T23:32:57.654 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:57 smithi076 ceph-mon[63913]: Upgrade: osd.5 is safe to restart 2024-09-13T23:32:57.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:57 smithi022 ceph-mon[80039]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:32:57.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:57.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T23:32:57.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:57 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T23:32:57.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:57 smithi022 ceph-mon[80039]: Upgrade: osd.5 is safe to restart 2024-09-13T23:32:57.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:57 smithi022 ceph-mon[82397]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:32:57.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:57.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T23:32:57.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:57 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T23:32:57.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:57 smithi022 ceph-mon[82397]: Upgrade: osd.5 is safe to restart 2024-09-13T23:32:58.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:58 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:58.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:58 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:58.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:58 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T23:32:58.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:58 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:32:58.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:58 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:58.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:58 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:58.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:58 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T23:32:58.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:58 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:32:58.693 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:58 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:32:58.693 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:58 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:32:58.693 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:58 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T23:32:58.693 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:58 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:32:59.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:59 smithi022 ceph-mon[80039]: Upgrade: Updating osd.5 2024-09-13T23:32:59.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:59 smithi022 ceph-mon[80039]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:32:59.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:32:59 smithi022 ceph-mon[80039]: Deploying daemon osd.5 on smithi076 2024-09-13T23:32:59.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:59 smithi022 ceph-mon[82397]: Upgrade: Updating osd.5 2024-09-13T23:32:59.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:59 smithi022 ceph-mon[82397]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:32:59.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:32:59 smithi022 ceph-mon[82397]: Deploying daemon osd.5 on smithi076 2024-09-13T23:32:59.687 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:59 smithi076 ceph-mon[63913]: Upgrade: Updating osd.5 2024-09-13T23:32:59.687 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:59 smithi076 ceph-mon[63913]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:32:59.687 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:32:59 smithi076 ceph-mon[63913]: Deploying daemon osd.5 on smithi076 2024-09-13T23:33:01.384 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:01 smithi076 systemd[1]: Stopping Ceph osd.5 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:33:01.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:01 smithi022 ceph-mon[80039]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:33:01.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:01 smithi022 ceph-mon[80039]: osd.5 marked itself down and dead 2024-09-13T23:33:01.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:01 smithi022 ceph-mon[82397]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:33:01.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:01 smithi022 ceph-mon[82397]: osd.5 marked itself down and dead 2024-09-13T23:33:01.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:01 smithi076 ceph-mon[63913]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:33:01.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:01 smithi076 ceph-mon[63913]: osd.5 marked itself down and dead 2024-09-13T23:33:01.788 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[30866]: 2024-09-13T23:33:01.382+0000 7f862fc31700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:33:01.788 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[30866]: 2024-09-13T23:33:01.382+0000 7f862fc31700 -1 osd.5 138 *** Got signal Terminated *** 2024-09-13T23:33:01.788 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:01 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[30866]: 2024-09-13T23:33:01.382+0000 7f862fc31700 -1 osd.5 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:33:02.108 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:01 smithi076 podman[78989]: 2024-09-13 23:33:01.822308367 +0000 UTC m=+0.551452714 container died 0e0c75069a99bc1c26d3c29e23ef3e2559b63b7db9f0cbe16730d5b17a944831 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, version=8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, RELEASE=HEAD, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True) 2024-09-13T23:33:02.377 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:02 smithi076 podman[78989]: 2024-09-13 23:33:02.107521496 +0000 UTC m=+0.836665833 container cleanup 0e0c75069a99bc1c26d3c29e23ef3e2559b63b7db9f0cbe16730d5b17a944831 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, distribution-scope=public, GIT_BRANCH=HEAD, vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.openshift.expose-services=, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-13T23:33:02.377 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:02 smithi076 bash[78989]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5 2024-09-13T23:33:02.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:33:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:33:02] "GET /metrics HTTP/1.1" 200 38133 "" "Prometheus/2.43.0" 2024-09-13T23:33:02.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:02 smithi022 ceph-mon[80039]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:33:02.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:02 smithi022 ceph-mon[80039]: osdmap e139: 8 total, 7 up, 8 in 2024-09-13T23:33:02.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:02 smithi022 ceph-mon[80039]: osdmap e140: 8 total, 7 up, 8 in 2024-09-13T23:33:02.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:02 smithi022 ceph-mon[82397]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:33:02.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:02 smithi022 ceph-mon[82397]: osdmap e139: 8 total, 7 up, 8 in 2024-09-13T23:33:02.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:02 smithi022 ceph-mon[82397]: osdmap e140: 8 total, 7 up, 8 in 2024-09-13T23:33:02.698 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:02 smithi076 ceph-mon[63913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:33:02.699 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:02 smithi076 ceph-mon[63913]: osdmap e139: 8 total, 7 up, 8 in 2024-09-13T23:33:02.699 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:02 smithi076 ceph-mon[63913]: osdmap e140: 8 total, 7 up, 8 in 2024-09-13T23:33:02.699 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:02 smithi076 podman[79005]: 2024-09-13 23:33:02.376187382 +0000 UTC m=+0.557046220 container remove 0e0c75069a99bc1c26d3c29e23ef3e2559b63b7db9f0cbe16730d5b17a944831 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, version=8, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, ceph=True, io.buildah.version=1.19.8, name=centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-13T23:33:02.952 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:02 smithi076 podman[79099]: 2024-09-13 23:33:02.600935904 +0000 UTC m=+0.016393355 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:33:02.952 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:02 smithi076 podman[79099]: 2024-09-13 23:33:02.702382369 +0000 UTC m=+0.117839852 container create a95183af6f110937db100c9795b47ad4a0b1acfff017f88f1019f2203da84ea3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T23:33:02.952 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:02 smithi076 podman[79099]: 2024-09-13 23:33:02.946941501 +0000 UTC m=+0.362398955 container init a95183af6f110937db100c9795b47ad4a0b1acfff017f88f1019f2203da84ea3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-13T23:33:03.259 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:02 smithi076 podman[79099]: 2024-09-13 23:33:02.951303937 +0000 UTC m=+0.366761386 container start a95183af6f110937db100c9795b47ad4a0b1acfff017f88f1019f2203da84ea3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-ecab7be) 2024-09-13T23:33:03.259 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:03 smithi076 podman[79099]: 2024-09-13 23:33:03.021942949 +0000 UTC m=+0.437400400 container attach a95183af6f110937db100c9795b47ad4a0b1acfff017f88f1019f2203da84ea3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-13T23:33:03.259 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:03 smithi076 podman[79099]: 2024-09-13 23:33:03.147060757 +0000 UTC m=+0.562518211 container died a95183af6f110937db100c9795b47ad4a0b1acfff017f88f1019f2203da84ea3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T23:33:03.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:03 smithi076 ceph-mon[63913]: pgmap v177: 161 pgs: 27 peering, 12 stale+active+clean, 122 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-09-13T23:33:03.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:03 smithi076 ceph-mon[63913]: Health check failed: Reduced data availability: 7 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-13T23:33:03.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:03 smithi022 ceph-mon[80039]: pgmap v177: 161 pgs: 27 peering, 12 stale+active+clean, 122 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-09-13T23:33:03.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:03 smithi022 ceph-mon[80039]: Health check failed: Reduced data availability: 7 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-13T23:33:03.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:03 smithi022 ceph-mon[82397]: pgmap v177: 161 pgs: 27 peering, 12 stale+active+clean, 122 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-09-13T23:33:03.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:03 smithi022 ceph-mon[82397]: Health check failed: Reduced data availability: 7 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-13T23:33:04.238 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:03 smithi076 podman[79123]: 2024-09-13 23:33:03.801467076 +0000 UTC m=+0.648647327 container remove a95183af6f110937db100c9795b47ad4a0b1acfff017f88f1019f2203da84ea3 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-13T23:33:04.239 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:03 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-13T23:33:04.239 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:03 smithi076 systemd[1]: Stopped Ceph osd.5 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:33:04.239 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:03 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.5.service: Consumed 15.083s CPU time. 2024-09-13T23:33:04.239 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:04 smithi076 systemd[1]: Starting Ceph osd.5 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:33:04.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:04 smithi076 ceph-mon[63913]: Health check failed: Degraded data redundancy: 19/627 objects degraded (3.030%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:04.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:04 smithi076 podman[79230]: 2024-09-13 23:33:04.140898201 +0000 UTC m=+0.016609370 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:33:04.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:04 smithi076 podman[79230]: 2024-09-13 23:33:04.270552078 +0000 UTC m=+0.146263247 container create e6c11f9a4a114451552e7229daa48a5e5f99686b58f476a8bb3cd87e6ca07767 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T23:33:04.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:04 smithi076 podman[79230]: 2024-09-13 23:33:04.480499043 +0000 UTC m=+0.356210215 container init e6c11f9a4a114451552e7229daa48a5e5f99686b58f476a8bb3cd87e6ca07767 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T23:33:04.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:04 smithi076 podman[79230]: 2024-09-13 23:33:04.483939855 +0000 UTC m=+0.359651027 container start e6c11f9a4a114451552e7229daa48a5e5f99686b58f476a8bb3cd87e6ca07767 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be) 2024-09-13T23:33:04.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:04 smithi022 ceph-mon[80039]: Health check failed: Degraded data redundancy: 19/627 objects degraded (3.030%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:04.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:04 smithi022 ceph-mon[82397]: Health check failed: Degraded data redundancy: 19/627 objects degraded (3.030%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:05.037 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:04 smithi076 podman[79230]: 2024-09-13 23:33:04.572312363 +0000 UTC m=+0.448023531 container attach e6c11f9a4a114451552e7229daa48a5e5f99686b58f476a8bb3cd87e6ca07767 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T23:33:05.037 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate[79245]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:33:05.038 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:04 smithi076 bash[79230]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:33:05.038 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:04 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate[79245]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:33:05.038 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:04 smithi076 bash[79230]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:33:05.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:05 smithi076 ceph-mon[63913]: pgmap v179: 161 pgs: 5 active+undersized, 34 peering, 5 stale+active+clean, 5 active+undersized+degraded, 112 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 19/627 objects degraded (3.030%) 2024-09-13T23:33:05.788 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate[79245]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T23:33:05.788 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 bash[79230]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T23:33:05.788 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate[79245]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T23:33:05.788 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 bash[79230]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T23:33:05.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:05 smithi022 ceph-mon[80039]: pgmap v179: 161 pgs: 5 active+undersized, 34 peering, 5 stale+active+clean, 5 active+undersized+degraded, 112 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 19/627 objects degraded (3.030%) 2024-09-13T23:33:05.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:05 smithi022 ceph-mon[82397]: pgmap v179: 161 pgs: 5 active+undersized, 34 peering, 5 stale+active+clean, 5 active+undersized+degraded, 112 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 19/627 objects degraded (3.030%) 2024-09-13T23:33:06.287 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate[79245]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T23:33:06.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 bash[79230]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T23:33:06.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate[79245]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T23:33:06.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 bash[79230]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T23:33:06.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate[79245]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T23:33:06.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 bash[79230]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T23:33:06.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate[79245]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T23:33:06.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 bash[79230]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T23:33:06.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate[79245]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-13T23:33:06.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 bash[79230]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-13T23:33:06.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:05 smithi076 podman[79230]: 2024-09-13 23:33:05.997876865 +0000 UTC m=+1.873588035 container died e6c11f9a4a114451552e7229daa48a5e5f99686b58f476a8bb3cd87e6ca07767 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True) 2024-09-13T23:33:06.921 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:06 smithi076 podman[79372]: 2024-09-13 23:33:06.618655391 +0000 UTC m=+0.614148164 container remove e6c11f9a4a114451552e7229daa48a5e5f99686b58f476a8bb3cd87e6ca07767 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T23:33:06.921 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:06 smithi076 podman[79432]: 2024-09-13 23:33:06.81961851 +0000 UTC m=+0.105958482 container create 57c6e1a1827dddd012c2ffe2fcf4c2c7114e0fe668d81223f5c49cdc671df2de (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-13T23:33:06.921 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:06 smithi076 podman[79432]: 2024-09-13 23:33:06.730159481 +0000 UTC m=+0.016499461 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:33:07.287 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:07 smithi076 podman[79432]: 2024-09-13 23:33:07.055408777 +0000 UTC m=+0.341748750 container init 57c6e1a1827dddd012c2ffe2fcf4c2c7114e0fe668d81223f5c49cdc671df2de (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-13T23:33:07.287 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:07 smithi076 podman[79432]: 2024-09-13 23:33:07.059039567 +0000 UTC m=+0.345379542 container start 57c6e1a1827dddd012c2ffe2fcf4c2c7114e0fe668d81223f5c49cdc671df2de (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-13T23:33:07.287 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:07 smithi076 ceph-osd[79452]: -- 172.21.15.76:0/1589655497 <== mon.2 v2:172.21.15.76:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x55db1b26bd40 con 0x55db1bff2000 2024-09-13T23:33:07.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:07 smithi076 ceph-osd[79452]: -- 172.21.15.76:0/1589655497 <== mon.2 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55db1b22ba40 con 0x55db1bff2000 2024-09-13T23:33:07.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:07 smithi076 bash[79432]: 57c6e1a1827dddd012c2ffe2fcf4c2c7114e0fe668d81223f5c49cdc671df2de 2024-09-13T23:33:07.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:07 smithi076 systemd[1]: Started Ceph osd.5 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:33:07.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:07.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:33:07.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:07 smithi022 ceph-mon[82397]: pgmap v180: 161 pgs: 12 active+undersized, 34 peering, 13 active+undersized+degraded, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 44/627 objects degraded (7.018%) 2024-09-13T23:33:07.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:07.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:07.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:07.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:33:07.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:07 smithi022 ceph-mon[80039]: pgmap v180: 161 pgs: 12 active+undersized, 34 peering, 13 active+undersized+degraded, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 44/627 objects degraded (7.018%) 2024-09-13T23:33:07.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:07.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:07.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:07.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:33:07.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:07 smithi076 ceph-mon[63913]: pgmap v180: 161 pgs: 12 active+undersized, 34 peering, 13 active+undersized+degraded, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 44/627 objects degraded (7.018%) 2024-09-13T23:33:07.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:07.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:08 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:08 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:08.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:08 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:09.037 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:08 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:33:08.928+0000 7f111e0ed640 -1 Falling back to public interface 2024-09-13T23:33:09.578 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:09 smithi022 ceph-mon[80039]: pgmap v181: 161 pgs: 38 active+undersized, 21 active+undersized+degraded, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 72/627 objects degraded (11.483%) 2024-09-13T23:33:09.578 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:09 smithi022 ceph-mon[82397]: pgmap v181: 161 pgs: 38 active+undersized, 21 active+undersized+degraded, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 72/627 objects degraded (11.483%) 2024-09-13T23:33:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:09 smithi076 ceph-mon[63913]: pgmap v181: 161 pgs: 38 active+undersized, 21 active+undersized+degraded, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 72/627 objects degraded (11.483%) 2024-09-13T23:33:10.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:10 smithi022 ceph-mon[80039]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 12 pgs peering) 2024-09-13T23:33:10.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:10 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 72/627 objects degraded (11.483%), 21 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:10.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:10 smithi022 ceph-mon[82397]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 12 pgs peering) 2024-09-13T23:33:10.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:10 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 72/627 objects degraded (11.483%), 21 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:10.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:10 smithi076 ceph-mon[63913]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 12 pgs peering) 2024-09-13T23:33:10.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:10 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 72/627 objects degraded (11.483%), 21 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:11 smithi022 ceph-mon[80039]: pgmap v182: 161 pgs: 38 active+undersized, 21 active+undersized+degraded, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 72/627 objects degraded (11.483%) 2024-09-13T23:33:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:11 smithi022 ceph-mon[82397]: pgmap v182: 161 pgs: 38 active+undersized, 21 active+undersized+degraded, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 72/627 objects degraded (11.483%) 2024-09-13T23:33:11.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:11 smithi076 ceph-mon[63913]: pgmap v182: 161 pgs: 38 active+undersized, 21 active+undersized+degraded, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 72/627 objects degraded (11.483%) 2024-09-13T23:33:12.787 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:33:12.467+0000 7f111e0ed640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-13T23:33:12.788 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:33:12.490+0000 7f111e0ed640 -1 osd.5 138 log_to_monitors true 2024-09-13T23:33:12.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:12 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:12.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:12 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:12.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:12 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:12.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:12 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:12.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:12 smithi076 ceph-mon[63913]: from='osd.5 [v2:172.21.15.76:6808/1391107203,v1:172.21.15.76:6809/1391107203]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T23:33:12.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:12 smithi076 ceph-mon[63913]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T23:33:12.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:33:12] "GET /metrics HTTP/1.1" 200 38200 "" "Prometheus/2.43.0" 2024-09-13T23:33:12.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:12.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:12.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:12.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:12.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-mon[80039]: from='osd.5 [v2:172.21.15.76:6808/1391107203,v1:172.21.15.76:6809/1391107203]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T23:33:12.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-mon[80039]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T23:33:12.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:12.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:12.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:12.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:12.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-mon[82397]: from='osd.5 [v2:172.21.15.76:6808/1391107203,v1:172.21.15.76:6809/1391107203]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T23:33:12.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:12 smithi022 ceph-mon[82397]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T23:33:13.874 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:33:13 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:33:13.618+0000 7f1111e90640 -1 osd.5 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:33:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:14 smithi022 ceph-mon[80039]: pgmap v183: 161 pgs: 38 active+undersized, 21 active+undersized+degraded, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 72/627 objects degraded (11.483%) 2024-09-13T23:33:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:14 smithi022 ceph-mon[80039]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T23:33:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:14 smithi022 ceph-mon[80039]: osdmap e141: 8 total, 7 up, 8 in 2024-09-13T23:33:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:14 smithi022 ceph-mon[80039]: from='osd.5 [v2:172.21.15.76:6808/1391107203,v1:172.21.15.76:6809/1391107203]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:33:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:14 smithi022 ceph-mon[80039]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:33:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:14 smithi022 ceph-mon[80039]: from='osd.5 ' entity='osd.5' 2024-09-13T23:33:14.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:14 smithi022 ceph-mon[82397]: pgmap v183: 161 pgs: 38 active+undersized, 21 active+undersized+degraded, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 72/627 objects degraded (11.483%) 2024-09-13T23:33:14.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:14 smithi022 ceph-mon[82397]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T23:33:14.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:14 smithi022 ceph-mon[82397]: osdmap e141: 8 total, 7 up, 8 in 2024-09-13T23:33:14.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:14 smithi022 ceph-mon[82397]: from='osd.5 [v2:172.21.15.76:6808/1391107203,v1:172.21.15.76:6809/1391107203]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:33:14.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:14 smithi022 ceph-mon[82397]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:33:14.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:14 smithi022 ceph-mon[82397]: from='osd.5 ' entity='osd.5' 2024-09-13T23:33:14.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:14 smithi076 ceph-mon[63913]: pgmap v183: 161 pgs: 38 active+undersized, 21 active+undersized+degraded, 102 active+clean; 585 KiB data, 198 MiB used, 715 GiB / 715 GiB avail; 72/627 objects degraded (11.483%) 2024-09-13T23:33:14.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:14 smithi076 ceph-mon[63913]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T23:33:14.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:14 smithi076 ceph-mon[63913]: osdmap e141: 8 total, 7 up, 8 in 2024-09-13T23:33:14.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:14 smithi076 ceph-mon[63913]: from='osd.5 [v2:172.21.15.76:6808/1391107203,v1:172.21.15.76:6809/1391107203]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:33:14.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:14 smithi076 ceph-mon[63913]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:33:14.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:14 smithi076 ceph-mon[63913]: from='osd.5 ' entity='osd.5' 2024-09-13T23:33:15.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:15 smithi022 ceph-mon[80039]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:33:15.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:15 smithi022 ceph-mon[80039]: osd.5 [v2:172.21.15.76:6808/1391107203,v1:172.21.15.76:6809/1391107203] boot 2024-09-13T23:33:15.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:15 smithi022 ceph-mon[80039]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T23:33:15.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:15 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:33:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:15 smithi022 ceph-mon[82397]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:33:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:15 smithi022 ceph-mon[82397]: osd.5 [v2:172.21.15.76:6808/1391107203,v1:172.21.15.76:6809/1391107203] boot 2024-09-13T23:33:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:15 smithi022 ceph-mon[82397]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T23:33:15.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:15 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:33:15.463 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:15 smithi076 ceph-mon[63913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:33:15.463 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:15 smithi076 ceph-mon[63913]: osd.5 [v2:172.21.15.76:6808/1391107203,v1:172.21.15.76:6809/1391107203] boot 2024-09-13T23:33:15.464 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:15 smithi076 ceph-mon[63913]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T23:33:15.464 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:15 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T23:33:15.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:33:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:33:15.655+0000 7fd7d6194640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-09-13T23:33:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[80039]: pgmap v185: 161 pgs: 38 active+undersized, 21 active+undersized+degraded, 102 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s; 72/627 objects degraded (11.483%) 2024-09-13T23:33:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:33:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:33:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:16.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[80039]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T23:33:16.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:33:16.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:16.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:16.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:16.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T23:33:16.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[82397]: pgmap v185: 161 pgs: 38 active+undersized, 21 active+undersized+degraded, 102 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s; 72/627 objects degraded (11.483%) 2024-09-13T23:33:16.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:16.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:16.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:33:16.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:33:16.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:16.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[82397]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T23:33:16.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:33:16.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:16.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:16.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:16.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:16 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T23:33:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:16 smithi076 ceph-mon[63913]: pgmap v185: 161 pgs: 38 active+undersized, 21 active+undersized+degraded, 102 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s; 72/627 objects degraded (11.483%) 2024-09-13T23:33:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:16.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:33:16.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:33:16.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:16.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:16 smithi076 ceph-mon[63913]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T23:33:16.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:33:16.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:16.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:16.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:16.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:16 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T23:33:17.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:17 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T23:33:17.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:17 smithi022 ceph-mon[80039]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-09-13T23:33:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:17 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T23:33:17.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:17 smithi022 ceph-mon[82397]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-09-13T23:33:17.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:17 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T23:33:17.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:17 smithi076 ceph-mon[63913]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-09-13T23:33:18.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:18 smithi022 ceph-mon[82397]: pgmap v188: 161 pgs: 5 peering, 36 active+undersized, 18 active+undersized+degraded, 102 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 63/627 objects degraded (10.048%) 2024-09-13T23:33:18.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:18 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 63/627 objects degraded (10.048%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:18.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:18 smithi022 ceph-mon[80039]: pgmap v188: 161 pgs: 5 peering, 36 active+undersized, 18 active+undersized+degraded, 102 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 63/627 objects degraded (10.048%) 2024-09-13T23:33:18.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:18 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 63/627 objects degraded (10.048%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:18.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:18 smithi076 ceph-mon[63913]: pgmap v188: 161 pgs: 5 peering, 36 active+undersized, 18 active+undersized+degraded, 102 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 63/627 objects degraded (10.048%) 2024-09-13T23:33:18.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:18 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 63/627 objects degraded (10.048%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:19 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:19.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:19 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:19 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:20.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:20 smithi076 ceph-mon[63913]: pgmap v189: 161 pgs: 5 peering, 4 active+undersized, 1 active+undersized+degraded, 151 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 1/627 objects degraded (0.159%) 2024-09-13T23:33:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:20 smithi022 ceph-mon[80039]: pgmap v189: 161 pgs: 5 peering, 4 active+undersized, 1 active+undersized+degraded, 151 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 1/627 objects degraded (0.159%) 2024-09-13T23:33:20.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:20 smithi022 ceph-mon[82397]: pgmap v189: 161 pgs: 5 peering, 4 active+undersized, 1 active+undersized+degraded, 151 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 1/627 objects degraded (0.159%) 2024-09-13T23:33:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:21 smithi076 ceph-mon[63913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded) 2024-09-13T23:33:21.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:21 smithi076 ceph-mon[63913]: Cluster is now healthy 2024-09-13T23:33:21.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:21 smithi022 ceph-mon[82397]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded) 2024-09-13T23:33:21.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:21 smithi022 ceph-mon[82397]: Cluster is now healthy 2024-09-13T23:33:21.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:21 smithi022 ceph-mon[80039]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded) 2024-09-13T23:33:21.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:21 smithi022 ceph-mon[80039]: Cluster is now healthy 2024-09-13T23:33:22.492 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:33:22.493 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:33:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:33:22] "GET /metrics HTTP/1.1" 200 38212 "" "Prometheus/2.43.0" 2024-09-13T23:33:22.493 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:22 smithi022 ceph-mon[80039]: pgmap v190: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 428 B/s rd, 0 op/s 2024-09-13T23:33:22.493 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:22.493 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:33:22.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:22 smithi022 ceph-mon[82397]: pgmap v190: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 428 B/s rd, 0 op/s 2024-09-13T23:33:22.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:22.494 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:33:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:22 smithi076 ceph-mon[63913]: pgmap v190: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 428 B/s rd, 0 op/s 2024-09-13T23:33:22.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:22.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:33:23.173 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:33:23.173 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (20m) 81s ago 28m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:33:23.173 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (7m) 11s ago 28m 82.6M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:33:23.173 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (7m) 81s ago 27m 58.7M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:33:23.173 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (7m) 11s ago 33m 441M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:33:23.173 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (17m) 81s ago 34m 503M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:33:23.173 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (5m) 81s ago 34m 53.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:33:23.173 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (6m) 11s ago 33m 50.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:33:23.173 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (5m) 81s ago 33m 42.5M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:33:23.173 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (20m) 81s ago 29m 16.1M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:33:23.174 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (20m) 11s ago 29m 16.2M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:33:23.174 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (3m) 81s ago 32m 51.8M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:33:23.174 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (3m) 81s ago 32m 52.8M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:33:23.174 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (2m) 81s ago 32m 48.6M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 42db9ab27cb1 2024-09-13T23:33:23.174 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (86s) 81s ago 31m 14.5M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 b891533f55d3 2024-09-13T23:33:23.174 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (50s) 11s ago 31m 51.6M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 ba143ef88b4e 2024-09-13T23:33:23.174 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (16s) 11s ago 30m 12.1M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 57c6e1a1827d 2024-09-13T23:33:23.174 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (30m) 11s ago 30m 59.6M 3248M 17.2.0 e1d6a67b021e e6456d57328b 2024-09-13T23:33:23.174 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (30m) 11s ago 30m 60.2M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:33:23.174 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (7m) 11s ago 29m 69.5M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:33:23.174 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (27m) 81s ago 27m 78.5M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:33:23.174 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (27m) 11s ago 27m 77.6M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:33:23.586 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:33:23.587 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:33:23.587 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:33:23.587 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:33:23.587 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:33:23.587 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:33:23.587 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:33:23.587 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:33:23.587 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-13T23:33:23.587 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 6 2024-09-13T23:33:23.587 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:33:23.587 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:33:23.587 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:33:23.588 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:33:23.588 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:33:23.588 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-13T23:33:23.588 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 11 2024-09-13T23:33:23.588 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:33:23.588 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:33:23.940 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:33:23.941 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:33:23.941 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:33:23.941 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-13T23:33:23.941 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:33:23.941 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "6/8 daemons upgraded", 2024-09-13T23:33:23.941 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T23:33:23.941 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:33:23.941 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:33:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:24 smithi076 ceph-mon[63913]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-09-13T23:33:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:24 smithi076 ceph-mon[63913]: from='client.54513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:24 smithi076 ceph-mon[63913]: from='client.54519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:24 smithi076 ceph-mon[63913]: from='client.64459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:24.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:24 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/78234029' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:24 smithi022 ceph-mon[80039]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-09-13T23:33:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:24 smithi022 ceph-mon[80039]: from='client.54513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:24 smithi022 ceph-mon[80039]: from='client.54519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:24 smithi022 ceph-mon[80039]: from='client.64459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:24.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:24 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/78234029' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:24.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:24 smithi022 ceph-mon[82397]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-09-13T23:33:24.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:24 smithi022 ceph-mon[82397]: from='client.54513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:24.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:24 smithi022 ceph-mon[82397]: from='client.54519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:24.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:24 smithi022 ceph-mon[82397]: from='client.64459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:24.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:24 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/78234029' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:25 smithi076 ceph-mon[63913]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:25.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:25 smithi022 ceph-mon[80039]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:25.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:25 smithi022 ceph-mon[82397]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:26 smithi076 ceph-mon[63913]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-09-13T23:33:26.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:26 smithi022 ceph-mon[80039]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-09-13T23:33:26.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:26 smithi022 ceph-mon[82397]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-09-13T23:33:28.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:28 smithi076 ceph-mon[63913]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-09-13T23:33:28.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:28 smithi022 ceph-mon[80039]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-09-13T23:33:28.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:28 smithi022 ceph-mon[82397]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-09-13T23:33:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:29 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:29 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:29.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:29 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:30 smithi076 ceph-mon[63913]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T23:33:30.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:30 smithi022 ceph-mon[82397]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T23:33:30.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:30 smithi022 ceph-mon[80039]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T23:33:31.348 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:31 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T23:33:31.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:31 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T23:33:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:31 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T23:33:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:32 smithi076 ceph-mon[63913]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:33:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:32 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T23:33:32.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:32 smithi076 ceph-mon[63913]: Upgrade: osd.6 is safe to restart 2024-09-13T23:33:32.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:32 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:32.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:32 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:32.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:32 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T23:33:32.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:32 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:33:32.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:33:32] "GET /metrics HTTP/1.1" 200 38212 "" "Prometheus/2.43.0" 2024-09-13T23:33:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[80039]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:33:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T23:33:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[80039]: Upgrade: osd.6 is safe to restart 2024-09-13T23:33:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T23:33:32.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:33:32.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[82397]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:33:32.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T23:33:32.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[82397]: Upgrade: osd.6 is safe to restart 2024-09-13T23:33:32.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:32.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:32.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T23:33:32.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:32 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:33:33.376 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:33 smithi076 ceph-mon[63913]: Upgrade: Updating osd.6 2024-09-13T23:33:33.376 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:33 smithi076 ceph-mon[63913]: Deploying daemon osd.6 on smithi076 2024-09-13T23:33:33.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:33 smithi022 ceph-mon[80039]: Upgrade: Updating osd.6 2024-09-13T23:33:33.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:33 smithi022 ceph-mon[80039]: Deploying daemon osd.6 on smithi076 2024-09-13T23:33:33.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:33 smithi022 ceph-mon[82397]: Upgrade: Updating osd.6 2024-09-13T23:33:33.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:33 smithi022 ceph-mon[82397]: Deploying daemon osd.6 on smithi076 2024-09-13T23:33:34.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:34 smithi076 ceph-mon[63913]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:33:34.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:34 smithi022 ceph-mon[80039]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:33:34.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:34 smithi022 ceph-mon[82397]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:33:35.172 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:34 smithi076 systemd[1]: Stopping Ceph osd.6 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:33:35.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:35 smithi076 ceph-mon[63913]: osd.6 marked itself down and dead 2024-09-13T23:33:35.537 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[33726]: 2024-09-13T23:33:35.169+0000 7fb1adc1d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:33:35.538 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[33726]: 2024-09-13T23:33:35.169+0000 7fb1adc1d700 -1 osd.6 143 *** Got signal Terminated *** 2024-09-13T23:33:35.538 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[33726]: 2024-09-13T23:33:35.169+0000 7fb1adc1d700 -1 osd.6 143 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:33:35.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:35 smithi022 ceph-mon[80039]: osd.6 marked itself down and dead 2024-09-13T23:33:35.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:35 smithi022 ceph-mon[82397]: osd.6 marked itself down and dead 2024-09-13T23:33:36.037 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:35 smithi076 podman[83778]: 2024-09-13 23:33:35.780092236 +0000 UTC m=+0.714761981 container died e6456d57328b919a5dee6ea55ea28280495f963547ae379af5b4d9870cc72acc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6, com.redhat.component=centos-stream-container, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.buildah.version=1.19.8, GIT_CLEAN=True, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, distribution-scope=public, name=centos-stream, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-13T23:33:36.430 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:36 smithi076 ceph-mon[63913]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:33:36.430 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:36 smithi076 ceph-mon[63913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:33:36.430 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:36 smithi076 ceph-mon[63913]: osdmap e144: 8 total, 7 up, 8 in 2024-09-13T23:33:36.431 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:36 smithi076 podman[83778]: 2024-09-13 23:33:36.154665505 +0000 UTC m=+1.089335292 container cleanup e6456d57328b919a5dee6ea55ea28280495f963547ae379af5b4d9870cc72acc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., ceph=True, io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.buildah.version=1.19.8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64) 2024-09-13T23:33:36.431 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:36 smithi076 bash[83778]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6 2024-09-13T23:33:36.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:36 smithi022 ceph-mon[80039]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:33:36.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:36 smithi022 ceph-mon[80039]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:33:36.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:36 smithi022 ceph-mon[80039]: osdmap e144: 8 total, 7 up, 8 in 2024-09-13T23:33:36.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:36 smithi022 ceph-mon[82397]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:33:36.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:36 smithi022 ceph-mon[82397]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:33:36.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:36 smithi022 ceph-mon[82397]: osdmap e144: 8 total, 7 up, 8 in 2024-09-13T23:33:36.739 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:36 smithi076 podman[83804]: 2024-09-13 23:33:36.429077031 +0000 UTC m=+0.651048868 container remove e6456d57328b919a5dee6ea55ea28280495f963547ae379af5b4d9870cc72acc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, architecture=x86_64, version=8) 2024-09-13T23:33:37.037 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:36 smithi076 podman[83892]: 2024-09-13 23:33:36.641970254 +0000 UTC m=+0.016040431 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:33:37.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:36 smithi076 podman[83892]: 2024-09-13 23:33:36.746591469 +0000 UTC m=+0.120661640 container create b7233951aa5a9eafdbb455bcaca016a5e0c0735dad133305ece222624d1ef3e2 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-13T23:33:37.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:36 smithi076 podman[83892]: 2024-09-13 23:33:36.973871869 +0000 UTC m=+0.347942051 container init b7233951aa5a9eafdbb455bcaca016a5e0c0735dad133305ece222624d1ef3e2 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T23:33:37.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:36 smithi076 podman[83892]: 2024-09-13 23:33:36.977573679 +0000 UTC m=+0.351643859 container start b7233951aa5a9eafdbb455bcaca016a5e0c0735dad133305ece222624d1ef3e2 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD) 2024-09-13T23:33:37.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:37 smithi076 ceph-mon[63913]: osdmap e145: 8 total, 7 up, 8 in 2024-09-13T23:33:37.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:37.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:33:37.538 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:37 smithi076 podman[83892]: 2024-09-13 23:33:37.068139595 +0000 UTC m=+0.442209769 container attach b7233951aa5a9eafdbb455bcaca016a5e0c0735dad133305ece222624d1ef3e2 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T23:33:37.538 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:37 smithi076 podman[83892]: 2024-09-13 23:33:37.165863037 +0000 UTC m=+0.539933209 container died b7233951aa5a9eafdbb455bcaca016a5e0c0735dad133305ece222624d1ef3e2 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-13T23:33:37.556 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:37 smithi022 ceph-mon[82397]: osdmap e145: 8 total, 7 up, 8 in 2024-09-13T23:33:37.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:37.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:33:37.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:37 smithi022 ceph-mon[80039]: osdmap e145: 8 total, 7 up, 8 in 2024-09-13T23:33:37.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:37.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:33:38.011 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:37 smithi076 podman[83916]: 2024-09-13 23:33:37.777830979 +0000 UTC m=+0.602947875 container remove b7233951aa5a9eafdbb455bcaca016a5e0c0735dad133305ece222624d1ef3e2 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD) 2024-09-13T23:33:38.012 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:37 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-13T23:33:38.012 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:37 smithi076 systemd[1]: Stopped Ceph osd.6 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:33:38.012 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:37 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.6.service: Consumed 14.004s CPU time. 2024-09-13T23:33:38.012 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:38 smithi076 systemd[1]: Starting Ceph osd.6 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:33:38.273 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:38 smithi076 podman[84017]: 2024-09-13 23:33:38.125234951 +0000 UTC m=+0.015919939 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:33:38.273 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:38 smithi076 podman[84017]: 2024-09-13 23:33:38.237017221 +0000 UTC m=+0.127702195 container create 37c6dc0deb24bce1a36c8f8cdb81fba0e4f82dcac2fbc526aad83197110dc9d9 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS) 2024-09-13T23:33:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:38 smithi076 ceph-mon[63913]: pgmap v200: 161 pgs: 3 peering, 11 stale+active+clean, 147 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T23:33:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:38 smithi076 ceph-mon[63913]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T23:33:38.538 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:38 smithi076 podman[84017]: 2024-09-13 23:33:38.456076758 +0000 UTC m=+0.346761731 container init 37c6dc0deb24bce1a36c8f8cdb81fba0e4f82dcac2fbc526aad83197110dc9d9 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be) 2024-09-13T23:33:38.538 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:38 smithi076 podman[84017]: 2024-09-13 23:33:38.459601747 +0000 UTC m=+0.350286722 container start 37c6dc0deb24bce1a36c8f8cdb81fba0e4f82dcac2fbc526aad83197110dc9d9 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:33:38.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:38 smithi022 ceph-mon[80039]: pgmap v200: 161 pgs: 3 peering, 11 stale+active+clean, 147 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T23:33:38.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:38 smithi022 ceph-mon[80039]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T23:33:38.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:38 smithi022 ceph-mon[82397]: pgmap v200: 161 pgs: 3 peering, 11 stale+active+clean, 147 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T23:33:38.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:38 smithi022 ceph-mon[82397]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T23:33:39.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:38 smithi076 podman[84017]: 2024-09-13 23:33:38.539292042 +0000 UTC m=+0.429977017 container attach 37c6dc0deb24bce1a36c8f8cdb81fba0e4f82dcac2fbc526aad83197110dc9d9 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True) 2024-09-13T23:33:39.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate[84033]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:33:39.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:38 smithi076 bash[84017]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:33:39.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate[84033]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:33:39.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:38 smithi076 bash[84017]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:33:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:39 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:39.537 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate[84033]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T23:33:39.538 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 bash[84017]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T23:33:39.538 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate[84033]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T23:33:39.538 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 bash[84017]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T23:33:39.578 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:39 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:39.578 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:39 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:39.978 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate[84033]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T23:33:39.978 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 bash[84017]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T23:33:39.978 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate[84033]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T23:33:39.978 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 bash[84017]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T23:33:39.978 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate[84033]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-13T23:33:39.978 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 bash[84017]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-13T23:33:39.978 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate[84033]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T23:33:39.978 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 bash[84017]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T23:33:39.978 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate[84033]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-13T23:33:39.978 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 bash[84017]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-13T23:33:39.978 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:39 smithi076 podman[84017]: 2024-09-13 23:33:39.68238609 +0000 UTC m=+1.573071079 container died 37c6dc0deb24bce1a36c8f8cdb81fba0e4f82dcac2fbc526aad83197110dc9d9 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:33:40.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:40 smithi076 ceph-mon[63913]: pgmap v201: 161 pgs: 21 active+undersized, 3 peering, 6 stale+active+clean, 14 active+undersized+degraded, 117 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 50/627 objects degraded (7.974%) 2024-09-13T23:33:40.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:40 smithi076 ceph-mon[63913]: Health check failed: Degraded data redundancy: 50/627 objects degraded (7.974%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:40.538 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:40 smithi076 podman[84158]: 2024-09-13 23:33:40.409366683 +0000 UTC m=+0.716205673 container remove 37c6dc0deb24bce1a36c8f8cdb81fba0e4f82dcac2fbc526aad83197110dc9d9 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-13T23:33:40.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:40 smithi022 ceph-mon[80039]: pgmap v201: 161 pgs: 21 active+undersized, 3 peering, 6 stale+active+clean, 14 active+undersized+degraded, 117 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 50/627 objects degraded (7.974%) 2024-09-13T23:33:40.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:40 smithi022 ceph-mon[80039]: Health check failed: Degraded data redundancy: 50/627 objects degraded (7.974%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:40.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:40 smithi022 ceph-mon[82397]: pgmap v201: 161 pgs: 21 active+undersized, 3 peering, 6 stale+active+clean, 14 active+undersized+degraded, 117 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 50/627 objects degraded (7.974%) 2024-09-13T23:33:40.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:40 smithi022 ceph-mon[82397]: Health check failed: Degraded data redundancy: 50/627 objects degraded (7.974%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:40.888 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:40 smithi076 podman[84215]: 2024-09-13 23:33:40.52135162 +0000 UTC m=+0.015900380 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:33:40.888 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:40 smithi076 podman[84215]: 2024-09-13 23:33:40.644300021 +0000 UTC m=+0.138848774 container create c0ab477f2b96c7a85e7a93983521ebdc1fad63611cff0bcc34dfd399289ed7ee (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T23:33:40.888 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:40 smithi076 podman[84215]: 2024-09-13 23:33:40.863050532 +0000 UTC m=+0.357599290 container init c0ab477f2b96c7a85e7a93983521ebdc1fad63611cff0bcc34dfd399289ed7ee (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T23:33:40.888 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:40 smithi076 podman[84215]: 2024-09-13 23:33:40.866293809 +0000 UTC m=+0.360842562 container start c0ab477f2b96c7a85e7a93983521ebdc1fad63611cff0bcc34dfd399289ed7ee (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:33:40.888 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:40 smithi076 ceph-osd[84234]: -- 172.21.15.76:0/1510119879 <== mon.2 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55835b7dba40 con 0x55835c5a2000 2024-09-13T23:33:41.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:40 smithi076 bash[84215]: c0ab477f2b96c7a85e7a93983521ebdc1fad63611cff0bcc34dfd399289ed7ee 2024-09-13T23:33:41.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:40 smithi076 systemd[1]: Started Ceph osd.6 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:33:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:42 smithi076 ceph-mon[63913]: pgmap v202: 161 pgs: 32 active+undersized, 3 peering, 18 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-09-13T23:33:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:42 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:42.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:42 smithi022 ceph-mon[82397]: pgmap v202: 161 pgs: 32 active+undersized, 3 peering, 18 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-09-13T23:33:42.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:42.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:42 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:42.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:42 smithi022 ceph-mon[80039]: pgmap v202: 161 pgs: 32 active+undersized, 3 peering, 18 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-09-13T23:33:42.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:42.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:42 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:42.787 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:33:42.516+0000 7fd13901a640 -1 Falling back to public interface 2024-09-13T23:33:42.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:33:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:33:42] "GET /metrics HTTP/1.1" 200 38215 "" "Prometheus/2.43.0" 2024-09-13T23:33:43.382 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:43 smithi076 ceph-mon[63913]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T23:33:43.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:43 smithi022 ceph-mon[80039]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T23:33:43.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:43 smithi022 ceph-mon[82397]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T23:33:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:44 smithi076 ceph-mon[63913]: pgmap v203: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-09-13T23:33:44.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:44 smithi022 ceph-mon[80039]: pgmap v203: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-09-13T23:33:44.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:44 smithi022 ceph-mon[82397]: pgmap v203: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-09-13T23:33:45.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:45 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 75/627 objects degraded (11.962%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:45.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:45.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:45.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:45 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 75/627 objects degraded (11.962%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:45.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:45.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:45 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 75/627 objects degraded (11.962%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:45.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:45.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:46.076 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:33:45.822+0000 7fd13901a640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-13T23:33:46.076 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:33:45.852+0000 7fd13901a640 -1 osd.6 143 log_to_monitors true 2024-09-13T23:33:46.326 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:46 smithi076 ceph-mon[63913]: pgmap v204: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-09-13T23:33:46.326 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:46 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:46.326 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:46 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:46.326 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:46 smithi076 ceph-mon[63913]: from='osd.6 [v2:172.21.15.76:6816/2558270779,v1:172.21.15.76:6817/2558270779]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T23:33:46.327 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:46 smithi076 ceph-mon[63913]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T23:33:46.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:46 smithi022 ceph-mon[80039]: pgmap v204: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-09-13T23:33:46.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:46 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:46.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:46 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:46.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:46 smithi022 ceph-mon[80039]: from='osd.6 [v2:172.21.15.76:6816/2558270779,v1:172.21.15.76:6817/2558270779]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T23:33:46.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:46 smithi022 ceph-mon[80039]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T23:33:46.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:46 smithi022 ceph-mon[82397]: pgmap v204: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-09-13T23:33:46.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:46 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:46.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:46 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:46.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:46 smithi022 ceph-mon[82397]: from='osd.6 [v2:172.21.15.76:6816/2558270779,v1:172.21.15.76:6817/2558270779]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T23:33:46.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:46 smithi022 ceph-mon[82397]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T23:33:46.964 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:33:46 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:33:46.614+0000 7fd130dc5640 -1 osd.6 143 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:33:47.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:47 smithi022 ceph-mon[80039]: pgmap v205: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-09-13T23:33:47.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:47 smithi022 ceph-mon[80039]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T23:33:47.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:47 smithi022 ceph-mon[80039]: osdmap e146: 8 total, 7 up, 8 in 2024-09-13T23:33:47.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:47 smithi022 ceph-mon[80039]: from='osd.6 [v2:172.21.15.76:6816/2558270779,v1:172.21.15.76:6817/2558270779]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:33:47.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:47 smithi022 ceph-mon[80039]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:33:47.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:47 smithi022 ceph-mon[82397]: pgmap v205: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-09-13T23:33:47.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:47 smithi022 ceph-mon[82397]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T23:33:47.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:47 smithi022 ceph-mon[82397]: osdmap e146: 8 total, 7 up, 8 in 2024-09-13T23:33:47.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:47 smithi022 ceph-mon[82397]: from='osd.6 [v2:172.21.15.76:6816/2558270779,v1:172.21.15.76:6817/2558270779]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:33:47.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:47 smithi022 ceph-mon[82397]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:33:47.966 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:47 smithi076 ceph-mon[63913]: pgmap v205: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-09-13T23:33:47.966 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:47 smithi076 ceph-mon[63913]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T23:33:47.966 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:47 smithi076 ceph-mon[63913]: osdmap e146: 8 total, 7 up, 8 in 2024-09-13T23:33:47.966 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:47 smithi076 ceph-mon[63913]: from='osd.6 [v2:172.21.15.76:6816/2558270779,v1:172.21.15.76:6817/2558270779]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:33:47.967 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:47 smithi076 ceph-mon[63913]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:33:48.890 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:48.890 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-mon[80039]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:33:48.890 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-mon[80039]: osd.6 [v2:172.21.15.76:6816/2558270779,v1:172.21.15.76:6817/2558270779] boot 2024-09-13T23:33:48.890 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-mon[80039]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T23:33:48.890 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:33:48.890 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-mon[80039]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T23:33:48.890 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:48.890 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-mon[82397]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:33:48.890 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-mon[82397]: osd.6 [v2:172.21.15.76:6816/2558270779,v1:172.21.15.76:6817/2558270779] boot 2024-09-13T23:33:48.891 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-mon[82397]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T23:33:48.891 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:33:48.891 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-mon[82397]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T23:33:49.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:48 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:49.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:48 smithi076 ceph-mon[63913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:33:49.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:48 smithi076 ceph-mon[63913]: osd.6 [v2:172.21.15.76:6816/2558270779,v1:172.21.15.76:6817/2558270779] boot 2024-09-13T23:33:49.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:48 smithi076 ceph-mon[63913]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T23:33:49.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:48 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T23:33:49.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:48 smithi076 ceph-mon[63913]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T23:33:49.177 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:33:48 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:33:48.887+0000 7fd7d6194640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-13T23:33:50.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: pgmap v208: 161 pgs: 17 peering, 24 active+undersized, 12 active+undersized+degraded, 108 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 52/627 objects degraded (8.293%) 2024-09-13T23:33:50.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:50.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:50.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:33:50.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:33:50.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:50.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:33:50.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:50.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:50.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:50.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T23:33:50.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T23:33:50.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:49 smithi076 ceph-mon[63913]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-13T23:33:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: pgmap v208: 161 pgs: 17 peering, 24 active+undersized, 12 active+undersized+degraded, 108 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 52/627 objects degraded (8.293%) 2024-09-13T23:33:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:33:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:33:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:50.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:33:50.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:50.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:50.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:50.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T23:33:50.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T23:33:50.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[80039]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-13T23:33:50.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: pgmap v208: 161 pgs: 17 peering, 24 active+undersized, 12 active+undersized+degraded, 108 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 52/627 objects degraded (8.293%) 2024-09-13T23:33:50.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:50.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:50.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:33:50.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:33:50.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:50.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:33:50.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:50.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:50.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:50.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T23:33:50.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T23:33:50.179 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:49 smithi022 ceph-mon[82397]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-13T23:33:51.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:50 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 52/627 objects degraded (8.293%), 12 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:51.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:50 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 52/627 objects degraded (8.293%), 12 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:51.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:50 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 52/627 objects degraded (8.293%), 12 pgs degraded (PG_DEGRADED) 2024-09-13T23:33:52.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:51 smithi076 ceph-mon[63913]: pgmap v210: 161 pgs: 17 peering, 18 active+undersized, 9 active+undersized+degraded, 117 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s; 38/627 objects degraded (6.061%) 2024-09-13T23:33:52.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:51 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:52.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:51 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:33:52.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:51 smithi022 ceph-mon[80039]: pgmap v210: 161 pgs: 17 peering, 18 active+undersized, 9 active+undersized+degraded, 117 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s; 38/627 objects degraded (6.061%) 2024-09-13T23:33:52.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:51 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:52.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:51 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:33:52.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:51 smithi022 ceph-mon[82397]: pgmap v210: 161 pgs: 17 peering, 18 active+undersized, 9 active+undersized+degraded, 117 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s; 38/627 objects degraded (6.061%) 2024-09-13T23:33:52.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:51 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:33:52.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:51 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:33:52.816 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:33:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:33:52] "GET /metrics HTTP/1.1" 200 38077 "" "Prometheus/2.43.0" 2024-09-13T23:33:53.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:52 smithi022 ceph-mon[80039]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/627 objects degraded (6.061%), 9 pgs degraded) 2024-09-13T23:33:53.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:52 smithi022 ceph-mon[80039]: Cluster is now healthy 2024-09-13T23:33:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:52 smithi022 ceph-mon[82397]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/627 objects degraded (6.061%), 9 pgs degraded) 2024-09-13T23:33:53.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:52 smithi022 ceph-mon[82397]: Cluster is now healthy 2024-09-13T23:33:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:52 smithi076 ceph-mon[63913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/627 objects degraded (6.061%), 9 pgs degraded) 2024-09-13T23:33:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:52 smithi076 ceph-mon[63913]: Cluster is now healthy 2024-09-13T23:33:54.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:53 smithi022 ceph-mon[80039]: pgmap v211: 161 pgs: 17 peering, 144 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-09-13T23:33:54.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:53 smithi022 ceph-mon[82397]: pgmap v211: 161 pgs: 17 peering, 144 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-09-13T23:33:54.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:53 smithi076 ceph-mon[63913]: pgmap v211: 161 pgs: 17 peering, 144 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-09-13T23:33:54.317 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:33:55.005 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:33:55.005 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (21m) 112s ago 29m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:33:55.005 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (7m) 10s ago 28m 81.7M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:33:55.005 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (8m) 112s ago 28m 58.7M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (8m) 10s ago 33m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (17m) 112s ago 35m 503M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (5m) 112s ago 35m 53.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (6m) 10s ago 33m 47.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (5m) 112s ago 34m 42.5M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (21m) 112s ago 29m 16.1M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (20m) 10s ago 29m 16.1M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (4m) 112s ago 33m 51.8M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (3m) 112s ago 32m 52.8M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (2m) 112s ago 32m 48.6M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 42db9ab27cb1 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (118s) 112s ago 32m 14.5M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 b891533f55d3 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (82s) 10s ago 31m 52.9M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 ba143ef88b4e 2024-09-13T23:33:55.006 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (48s) 10s ago 31m 53.0M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 57c6e1a1827d 2024-09-13T23:33:55.007 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (14s) 10s ago 31m 14.3M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 c0ab477f2b96 2024-09-13T23:33:55.007 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (30m) 10s ago 30m 60.6M 3248M 17.2.0 e1d6a67b021e 901ec98ff743 2024-09-13T23:33:55.007 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (8m) 10s ago 29m 69.5M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:33:55.007 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (28m) 112s ago 28m 78.5M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:33:55.007 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (28m) 10s ago 28m 77.8M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:33:55.423 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:33:55.423 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:33:55.423 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 7 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:33:55.424 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-13T23:33:55.425 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 12 2024-09-13T23:33:55.425 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:33:55.425 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:33:55.776 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:33:55.776 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:33:55.776 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:33:55.776 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-13T23:33:55.776 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:33:55.776 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "7/8 daemons upgraded", 2024-09-13T23:33:55.776 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T23:33:55.776 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:33:55.776 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:33:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:55 smithi022 ceph-mon[80039]: from='client.64477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:55 smithi022 ceph-mon[80039]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:33:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:55 smithi022 ceph-mon[80039]: from='client.64483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:55 smithi022 ceph-mon[80039]: from='client.54555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:55 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/3919563335' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:56.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:55 smithi022 ceph-mon[82397]: from='client.64477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:56.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:55 smithi022 ceph-mon[82397]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:33:56.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:55 smithi022 ceph-mon[82397]: from='client.64483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:56.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:55 smithi022 ceph-mon[82397]: from='client.54555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:56.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:55 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/3919563335' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:55 smithi076 ceph-mon[63913]: from='client.64477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:55 smithi076 ceph-mon[63913]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:33:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:55 smithi076 ceph-mon[63913]: from='client.64483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:56.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:55 smithi076 ceph-mon[63913]: from='client.54555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:56.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:55 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/3919563335' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:33:57.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:56 smithi022 ceph-mon[80039]: from='client.64501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:57.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:56 smithi022 ceph-mon[82397]: from='client.64501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:57.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:56 smithi076 ceph-mon[63913]: from='client.64501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:33:58.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:57 smithi022 ceph-mon[80039]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T23:33:58.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:57 smithi022 ceph-mon[82397]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T23:33:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:57 smithi076 ceph-mon[63913]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T23:33:59.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:58 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:59.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:58 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:33:59.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:58 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:00.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:33:59 smithi022 ceph-mon[80039]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:34:00.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:33:59 smithi022 ceph-mon[82397]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:34:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:33:59 smithi076 ceph-mon[63913]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:34:02.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:01 smithi022 ceph-mon[80039]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 871 B/s rd, 0 op/s 2024-09-13T23:34:02.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:01 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:02.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:01 smithi022 ceph-mon[82397]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 871 B/s rd, 0 op/s 2024-09-13T23:34:02.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:01 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:01 smithi076 ceph-mon[63913]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 871 B/s rd, 0 op/s 2024-09-13T23:34:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:01 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:02.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:34:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:34:02] "GET /metrics HTTP/1.1" 200 38077 "" "Prometheus/2.43.0" 2024-09-13T23:34:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:03 smithi022 ceph-mon[80039]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:04.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:03 smithi022 ceph-mon[82397]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:03 smithi076 ceph-mon[63913]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:05.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:04 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T23:34:05.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:04 smithi022 ceph-mon[80039]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T23:34:05.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:04 smithi022 ceph-mon[80039]: Upgrade: osd.7 is safe to restart 2024-09-13T23:34:05.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:04 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T23:34:05.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:04 smithi022 ceph-mon[82397]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T23:34:05.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:04 smithi022 ceph-mon[82397]: Upgrade: osd.7 is safe to restart 2024-09-13T23:34:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:04 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T23:34:05.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:04 smithi076 ceph-mon[63913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T23:34:05.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:04 smithi076 ceph-mon[63913]: Upgrade: osd.7 is safe to restart 2024-09-13T23:34:06.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:05 smithi076 ceph-mon[63913]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:06.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:05 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:06.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:05 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T23:34:06.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:05 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:05 smithi022 ceph-mon[80039]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:05 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:05 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T23:34:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:05 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:06.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:05 smithi022 ceph-mon[82397]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:06.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:05 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:06.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:05 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T23:34:06.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:05 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:07.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:06 smithi076 ceph-mon[63913]: Upgrade: Updating osd.7 2024-09-13T23:34:07.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:06 smithi076 ceph-mon[63913]: Deploying daemon osd.7 on smithi076 2024-09-13T23:34:07.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:07.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:34:07.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:06 smithi022 ceph-mon[80039]: Upgrade: Updating osd.7 2024-09-13T23:34:07.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:06 smithi022 ceph-mon[80039]: Deploying daemon osd.7 on smithi076 2024-09-13T23:34:07.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:07.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:34:07.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:06 smithi022 ceph-mon[82397]: Upgrade: Updating osd.7 2024-09-13T23:34:07.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:06 smithi022 ceph-mon[82397]: Deploying daemon osd.7 on smithi076 2024-09-13T23:34:07.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:07.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:34:07.990 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:07 smithi076 ceph-mon[63913]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:08.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:07 smithi022 ceph-mon[80039]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:07 smithi022 ceph-mon[82397]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:08.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:08 smithi076 systemd[1]: Stopping Ceph osd.7 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:34:09.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:08 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:09.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:08 smithi076 ceph-mon[63913]: osd.7 marked itself down and dead 2024-09-13T23:34:09.038 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:08 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[36633]: 2024-09-13T23:34:08.627+0000 7f1df0745700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:34:09.038 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:08 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[36633]: 2024-09-13T23:34:08.627+0000 7f1df0745700 -1 osd.7 148 *** Got signal Terminated *** 2024-09-13T23:34:09.038 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:08 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[36633]: 2024-09-13T23:34:08.627+0000 7f1df0745700 -1 osd.7 148 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:34:09.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:08 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:09.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:08 smithi022 ceph-mon[80039]: osd.7 marked itself down and dead 2024-09-13T23:34:09.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:08 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:09.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:08 smithi022 ceph-mon[82397]: osd.7 marked itself down and dead 2024-09-13T23:34:09.504 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:09 smithi076 podman[88275]: 2024-09-13 23:34:09.210191396 +0000 UTC m=+0.699231991 container died 901ec98ff74351930de1120f2fe50f8293a6ed8d610e40078e3b9041e4a88412 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, ceph=True, name=centos-stream, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, io.buildah.version=1.19.8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64) 2024-09-13T23:34:09.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:09 smithi076 podman[88275]: 2024-09-13 23:34:09.602231353 +0000 UTC m=+1.091271947 container cleanup 901ec98ff74351930de1120f2fe50f8293a6ed8d610e40078e3b9041e4a88412 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream) 2024-09-13T23:34:09.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:09 smithi076 bash[88275]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7 2024-09-13T23:34:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:09 smithi022 ceph-mon[80039]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:09 smithi022 ceph-mon[80039]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:34:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:09 smithi022 ceph-mon[80039]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T23:34:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:09 smithi022 ceph-mon[80039]: osdmap e149: 8 total, 7 up, 8 in 2024-09-13T23:34:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:09 smithi022 ceph-mon[82397]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:09 smithi022 ceph-mon[82397]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:34:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:09 smithi022 ceph-mon[82397]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T23:34:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:09 smithi022 ceph-mon[82397]: osdmap e149: 8 total, 7 up, 8 in 2024-09-13T23:34:10.186 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:09 smithi076 ceph-mon[63913]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:10.186 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:09 smithi076 ceph-mon[63913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T23:34:10.186 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:09 smithi076 ceph-mon[63913]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T23:34:10.186 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:09 smithi076 ceph-mon[63913]: osdmap e149: 8 total, 7 up, 8 in 2024-09-13T23:34:10.186 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:09 smithi076 podman[88289]: 2024-09-13 23:34:09.845020625 +0000 UTC m=+0.637629545 container remove 901ec98ff74351930de1120f2fe50f8293a6ed8d610e40078e3b9041e4a88412 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7, release=754, build-date=2022-05-03T08:36:31.336870, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.openshift.expose-services=, com.redhat.component=centos-stream-container, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, architecture=x86_64, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, distribution-scope=public, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-13T23:34:10.524 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:10 smithi076 podman[88379]: 2024-09-13 23:34:10.088284953 +0000 UTC m=+0.016096546 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:34:10.524 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:10 smithi076 podman[88379]: 2024-09-13 23:34:10.187503711 +0000 UTC m=+0.115315291 container create 0900cc80a2212a0d9111f8a087bccb06045af39bdd124bcc8215c0ad030aaf85 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-13T23:34:10.524 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:10 smithi076 podman[88379]: 2024-09-13 23:34:10.431617918 +0000 UTC m=+0.359429498 container init 0900cc80a2212a0d9111f8a087bccb06045af39bdd124bcc8215c0ad030aaf85 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:34:10.524 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:10 smithi076 podman[88379]: 2024-09-13 23:34:10.43500589 +0000 UTC m=+0.362817469 container start 0900cc80a2212a0d9111f8a087bccb06045af39bdd124bcc8215c0ad030aaf85 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=) 2024-09-13T23:34:10.524 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:10 smithi076 podman[88379]: 2024-09-13 23:34:10.523420686 +0000 UTC m=+0.451232274 container attach 0900cc80a2212a0d9111f8a087bccb06045af39bdd124bcc8215c0ad030aaf85 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-09-13T23:34:10.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:10 smithi076 podman[88379]: 2024-09-13 23:34:10.620882907 +0000 UTC m=+0.548694497 container died 0900cc80a2212a0d9111f8a087bccb06045af39bdd124bcc8215c0ad030aaf85 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:34:11.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:10 smithi022 ceph-mon[80039]: osdmap e150: 8 total, 7 up, 8 in 2024-09-13T23:34:11.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:10 smithi022 ceph-mon[82397]: osdmap e150: 8 total, 7 up, 8 in 2024-09-13T23:34:11.190 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:10 smithi076 ceph-mon[63913]: osdmap e150: 8 total, 7 up, 8 in 2024-09-13T23:34:11.446 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:11 smithi076 podman[88402]: 2024-09-13 23:34:11.220429286 +0000 UTC m=+0.593174391 container remove 0900cc80a2212a0d9111f8a087bccb06045af39bdd124bcc8215c0ad030aaf85 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be) 2024-09-13T23:34:11.446 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:11 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-13T23:34:11.446 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:11 smithi076 systemd[1]: Stopped Ceph osd.7 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:34:11.446 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:11 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.7.service: Consumed 16.116s CPU time. 2024-09-13T23:34:11.720 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:11 smithi076 systemd[1]: Starting Ceph osd.7 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:34:11.720 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:11 smithi076 podman[88500]: 2024-09-13 23:34:11.558266508 +0000 UTC m=+0.015961813 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:34:12.014 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:11 smithi076 ceph-mon[63913]: pgmap v222: 161 pgs: 10 peering, 16 stale+active+clean, 135 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-09-13T23:34:12.014 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:11 smithi076 ceph-mon[63913]: Health check failed: Reduced data availability: 1 pg inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T23:34:12.014 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:11 smithi076 podman[88500]: 2024-09-13 23:34:11.719636857 +0000 UTC m=+0.177332207 container create 2693f7dc932ad00643806a31cb1b2029e39c2a5fd677f33832f1ff703f70bc2a (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T23:34:12.014 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:11 smithi076 podman[88500]: 2024-09-13 23:34:11.929921539 +0000 UTC m=+0.387616840 container init 2693f7dc932ad00643806a31cb1b2029e39c2a5fd677f33832f1ff703f70bc2a (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T23:34:12.014 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:11 smithi076 podman[88500]: 2024-09-13 23:34:11.933688057 +0000 UTC m=+0.391383359 container start 2693f7dc932ad00643806a31cb1b2029e39c2a5fd677f33832f1ff703f70bc2a (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-13T23:34:12.014 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 podman[88500]: 2024-09-13 23:34:12.013359446 +0000 UTC m=+0.471054761 container attach 2693f7dc932ad00643806a31cb1b2029e39c2a5fd677f33832f1ff703f70bc2a (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:34:12.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:11 smithi022 ceph-mon[80039]: pgmap v222: 161 pgs: 10 peering, 16 stale+active+clean, 135 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-09-13T23:34:12.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:11 smithi022 ceph-mon[80039]: Health check failed: Reduced data availability: 1 pg inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T23:34:12.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:11 smithi022 ceph-mon[82397]: pgmap v222: 161 pgs: 10 peering, 16 stale+active+clean, 135 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-09-13T23:34:12.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:11 smithi022 ceph-mon[82397]: Health check failed: Reduced data availability: 1 pg inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T23:34:12.287 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate[88515]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:34:12.287 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 bash[88500]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:34:12.288 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate[88515]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:34:12.288 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 bash[88500]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T23:34:12.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate[88515]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T23:34:12.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 bash[88500]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T23:34:12.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate[88515]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T23:34:12.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 bash[88500]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T23:34:12.926 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:34:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:34:12] "GET /metrics HTTP/1.1" 200 38229 "" "Prometheus/2.43.0" 2024-09-13T23:34:13.216 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:12 smithi076 ceph-mon[63913]: Health check failed: Degraded data redundancy: 15/627 objects degraded (2.392%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T23:34:13.216 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate[88515]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T23:34:13.216 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 bash[88500]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T23:34:13.216 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate[88515]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T23:34:13.216 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 bash[88500]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T23:34:13.216 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate[88515]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-13T23:34:13.216 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 bash[88500]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-13T23:34:13.217 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate[88515]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T23:34:13.217 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 bash[88500]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T23:34:13.217 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate[88515]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-13T23:34:13.217 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 bash[88500]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-13T23:34:13.217 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:12 smithi076 podman[88500]: 2024-09-13 23:34:12.854338207 +0000 UTC m=+1.312033512 container died 2693f7dc932ad00643806a31cb1b2029e39c2a5fd677f33832f1ff703f70bc2a (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be) 2024-09-13T23:34:13.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:12 smithi022 ceph-mon[82397]: Health check failed: Degraded data redundancy: 15/627 objects degraded (2.392%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T23:34:13.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:12 smithi022 ceph-mon[80039]: Health check failed: Degraded data redundancy: 15/627 objects degraded (2.392%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T23:34:13.868 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:13 smithi076 podman[88645]: 2024-09-13 23:34:13.559289885 +0000 UTC m=+0.699191743 container remove 2693f7dc932ad00643806a31cb1b2029e39c2a5fd677f33832f1ff703f70bc2a (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:34:13.868 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:13 smithi076 podman[88701]: 2024-09-13 23:34:13.760129506 +0000 UTC m=+0.107983650 container create 4325dc4e4077936bcebbad7c5d10ebcc82a6bb91c84948eb086105f63242dc20 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-13T23:34:13.869 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:13 smithi076 podman[88701]: 2024-09-13 23:34:13.668626379 +0000 UTC m=+0.016480534 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:34:14.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:13 smithi076 ceph-mon[63913]: pgmap v223: 161 pgs: 20 active+undersized, 10 peering, 7 stale+active+clean, 9 active+undersized+degraded, 115 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 15/627 objects degraded (2.392%) 2024-09-13T23:34:14.287 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:13 smithi076 podman[88701]: 2024-09-13 23:34:13.970684328 +0000 UTC m=+0.318538482 container init 4325dc4e4077936bcebbad7c5d10ebcc82a6bb91c84948eb086105f63242dc20 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True) 2024-09-13T23:34:14.288 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:13 smithi076 podman[88701]: 2024-09-13 23:34:13.974166576 +0000 UTC m=+0.322020722 container start 4325dc4e4077936bcebbad7c5d10ebcc82a6bb91c84948eb086105f63242dc20 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be) 2024-09-13T23:34:14.288 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:13 smithi076 ceph-osd[88720]: -- 172.21.15.76:0/1273311491 <== mon.2 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d72d783a40 con 0x55d72d761c00 2024-09-13T23:34:14.288 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:14 smithi076 bash[88701]: 4325dc4e4077936bcebbad7c5d10ebcc82a6bb91c84948eb086105f63242dc20 2024-09-13T23:34:14.288 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:14 smithi076 systemd[1]: Started Ceph osd.7 for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:34:14.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:13 smithi022 ceph-mon[80039]: pgmap v223: 161 pgs: 20 active+undersized, 10 peering, 7 stale+active+clean, 9 active+undersized+degraded, 115 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 15/627 objects degraded (2.392%) 2024-09-13T23:34:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:13 smithi022 ceph-mon[82397]: pgmap v223: 161 pgs: 20 active+undersized, 10 peering, 7 stale+active+clean, 9 active+undersized+degraded, 115 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 15/627 objects degraded (2.392%) 2024-09-13T23:34:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:15 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:15.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:15 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:15.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:15 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:15.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:15 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:15.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:15 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:15.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:15 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:16.037 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:15 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:34:15.615+0000 7fe9f1d35640 -1 Falling back to public interface 2024-09-13T23:34:16.414 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:16 smithi076 ceph-mon[63913]: pgmap v224: 161 pgs: 38 active+undersized, 10 peering, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 50/627 objects degraded (7.974%) 2024-09-13T23:34:16.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:16 smithi022 ceph-mon[82397]: pgmap v224: 161 pgs: 38 active+undersized, 10 peering, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 50/627 objects degraded (7.974%) 2024-09-13T23:34:16.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:16 smithi022 ceph-mon[80039]: pgmap v224: 161 pgs: 38 active+undersized, 10 peering, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 50/627 objects degraded (7.974%) 2024-09-13T23:34:17.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:17 smithi076 ceph-mon[63913]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 6 pgs peering) 2024-09-13T23:34:17.588 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:17 smithi022 ceph-mon[80039]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 6 pgs peering) 2024-09-13T23:34:17.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:17 smithi022 ceph-mon[82397]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 6 pgs peering) 2024-09-13T23:34:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:18 smithi076 ceph-mon[63913]: pgmap v225: 161 pgs: 41 active+undersized, 28 active+undersized+degraded, 92 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-09-13T23:34:18.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:18 smithi022 ceph-mon[80039]: pgmap v225: 161 pgs: 41 active+undersized, 28 active+undersized+degraded, 92 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-09-13T23:34:18.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:18 smithi022 ceph-mon[82397]: pgmap v225: 161 pgs: 41 active+undersized, 28 active+undersized+degraded, 92 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-09-13T23:34:19.419 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:19 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:34:19.126+0000 7fe9f1d35640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-13T23:34:19.419 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:19 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:34:19.154+0000 7fe9f1d35640 -1 osd.7 148 log_to_monitors true 2024-09-13T23:34:19.420 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:19 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:19.420 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:19 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:19.420 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:19 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:19.420 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:19 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:19.420 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:19 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:19.420 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:19 smithi076 ceph-mon[63913]: from='osd.7 [v2:172.21.15.76:6824/2919945764,v1:172.21.15.76:6825/2919945764]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T23:34:19.420 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:19 smithi076 ceph-mon[63913]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T23:34:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[82397]: from='osd.7 [v2:172.21.15.76:6824/2919945764,v1:172.21.15.76:6825/2919945764]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T23:34:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[82397]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T23:34:19.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:19.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:19.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:19.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:19.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:19.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[80039]: from='osd.7 [v2:172.21.15.76:6824/2919945764,v1:172.21.15.76:6825/2919945764]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T23:34:19.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:19 smithi022 ceph-mon[80039]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T23:34:20.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:34:20 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:34:20.360+0000 7fe9e5ad8640 -1 osd.7 148 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T23:34:20.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:20 smithi076 ceph-mon[63913]: pgmap v226: 161 pgs: 41 active+undersized, 28 active+undersized+degraded, 92 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-09-13T23:34:20.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:20 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T23:34:20.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:20 smithi076 ceph-mon[63913]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T23:34:20.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:20 smithi076 ceph-mon[63913]: from='osd.7 [v2:172.21.15.76:6824/2919945764,v1:172.21.15.76:6825/2919945764]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:34:20.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:20 smithi076 ceph-mon[63913]: osdmap e151: 8 total, 7 up, 8 in 2024-09-13T23:34:20.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:20 smithi076 ceph-mon[63913]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:34:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:20 smithi022 ceph-mon[80039]: pgmap v226: 161 pgs: 41 active+undersized, 28 active+undersized+degraded, 92 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-09-13T23:34:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:20 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T23:34:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:20 smithi022 ceph-mon[80039]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T23:34:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:20 smithi022 ceph-mon[80039]: from='osd.7 [v2:172.21.15.76:6824/2919945764,v1:172.21.15.76:6825/2919945764]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:34:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:20 smithi022 ceph-mon[80039]: osdmap e151: 8 total, 7 up, 8 in 2024-09-13T23:34:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:20 smithi022 ceph-mon[80039]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:34:20.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:20 smithi022 ceph-mon[82397]: pgmap v226: 161 pgs: 41 active+undersized, 28 active+undersized+degraded, 92 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-09-13T23:34:20.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:20 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T23:34:20.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:20 smithi022 ceph-mon[82397]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T23:34:20.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:20 smithi022 ceph-mon[82397]: from='osd.7 [v2:172.21.15.76:6824/2919945764,v1:172.21.15.76:6825/2919945764]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:34:20.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:20 smithi022 ceph-mon[82397]: osdmap e151: 8 total, 7 up, 8 in 2024-09-13T23:34:20.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:20 smithi022 ceph-mon[82397]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-13T23:34:21.652 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:21 smithi076 ceph-mon[63913]: pgmap v228: 161 pgs: 41 active+undersized, 28 active+undersized+degraded, 92 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-09-13T23:34:21.652 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:21 smithi076 ceph-mon[63913]: from='osd.7 ' entity='osd.7' 2024-09-13T23:34:21.652 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:21.652 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:34:21.676 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:21 smithi022 ceph-mon[80039]: pgmap v228: 161 pgs: 41 active+undersized, 28 active+undersized+degraded, 92 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-09-13T23:34:21.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:21 smithi022 ceph-mon[80039]: from='osd.7 ' entity='osd.7' 2024-09-13T23:34:21.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:21.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:34:21.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:21 smithi022 ceph-mon[82397]: pgmap v228: 161 pgs: 41 active+undersized, 28 active+undersized+degraded, 92 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-09-13T23:34:21.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:21 smithi022 ceph-mon[82397]: from='osd.7 ' entity='osd.7' 2024-09-13T23:34:21.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:21.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:34:22.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:34:22] "GET /metrics HTTP/1.1" 200 38164 "" "Prometheus/2.43.0" 2024-09-13T23:34:22.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:34:22.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: osd.7 [v2:172.21.15.76:6824/2919945764,v1:172.21.15.76:6825/2919945764] boot 2024-09-13T23:34:22.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T23:34:22.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:34:22.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:22.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:22.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:22.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:34:22.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all osd 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-13T23:34:22.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T23:34:22.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-13T23:34:22.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T23:34:22.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-13T23:34:22.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T23:34:22.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-13T23:34:22.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:34:22.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: osd.7 [v2:172.21.15.76:6824/2919945764,v1:172.21.15.76:6825/2919945764] boot 2024-09-13T23:34:22.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T23:34:22.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:34:22.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:22.679 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:22.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:22.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:34:22.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:22.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:34:22.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all osd 2024-09-13T23:34:22.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:22.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-13T23:34:22.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T23:34:22.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-13T23:34:22.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T23:34:22.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-13T23:34:22.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T23:34:22.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-13T23:34:22.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T23:34:22.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-13T23:34:22.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T23:34:22.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-13T23:34:22.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T23:34:22.682 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-13T23:34:22.682 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T23:34:22.682 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-13T23:34:22.682 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T23:34:22.682 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T23:34:22.682 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-13T23:34:22.682 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T23:34:22.682 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T23:34:22.682 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T23:34:22.682 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T23:34:22.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-13T23:34:22.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T23:34:22.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-13T23:34:22.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T23:34:22.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T23:34:22.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-13T23:34:22.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T23:34:22.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T23:34:22.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T23:34:22.683 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T23:34:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: osd.7 [v2:172.21.15.76:6824/2919945764,v1:172.21.15.76:6825/2919945764] boot 2024-09-13T23:34:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T23:34:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T23:34:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:34:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:34:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all osd 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T23:34:22.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T23:34:22.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: pgmap v230: 161 pgs: 10 peering, 35 active+undersized, 24 active+undersized+degraded, 92 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 78/627 objects degraded (12.440%) 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mds 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all nfs 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.678 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: Upgrade: Finalizing container_image settings 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:34:23.679 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:34:23.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:34:23.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: pgmap v230: 161 pgs: 10 peering, 35 active+undersized, 24 active+undersized+degraded, 92 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 78/627 objects degraded (12.440%) 2024-09-13T23:34:23.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mds 2024-09-13T23:34:23.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:34:23.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:34:23.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all nfs 2024-09-13T23:34:23.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:34:23.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.681 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: Upgrade: Finalizing container_image settings 2024-09-13T23:34:23.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:34:23.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:34:23.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:34:23.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:34:23.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:34:23.683 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:34:23.683 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T23:34:23.683 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:34:23.683 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:34:23.683 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:34:23.683 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:34:23.683 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:34:23.683 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.683 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:34:23.683 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:34:23.683 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:34:23.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:34:23.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:34:23.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:34:23.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:34:23.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: Upgrade: Complete! 2024-09-13T23:34:23.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:34:23.685 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:34:23.685 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T23:34:23.685 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:34:23.685 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:34:23.685 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:34:23.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:34:23.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:34:23.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:34:23.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:34:23.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:34:23.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:34:23.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:34:23.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:34:23.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:34:23.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.686 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.687 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.687 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.687 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.687 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.687 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: Upgrade: Complete! 2024-09-13T23:34:23.687 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:34:23.687 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:34:23.687 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:23.687 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:34:23.687 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.688 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:23.688 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:34:23.688 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.688 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:34:23.688 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:23.688 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:34:23.688 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.688 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:23.688 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:34:23.689 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:23 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: pgmap v230: 161 pgs: 10 peering, 35 active+undersized, 24 active+undersized+degraded, 92 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 78/627 objects degraded (12.440%) 2024-09-13T23:34:23.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mds 2024-09-13T23:34:23.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:34:23.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all nfs 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: Upgrade: Finalizing container_image settings 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T23:34:23.791 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:34:23.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:34:23.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:34:23.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:34:23.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:34:23.792 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:34:23.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:34:23.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:34:23.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:34:23.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:34:23.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:34:23.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:34:23.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.793 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:34:23.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: Upgrade: Complete! 2024-09-13T23:34:23.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:34:23.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:34:23.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:23.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:34:23.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:23.794 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:23.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:34:23.795 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:23 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:25.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:25 smithi022 ceph-mon[80039]: pgmap v232: 161 pgs: 4 activating, 10 peering, 21 active+undersized, 15 active+undersized+degraded, 111 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/627 objects degraded (8.931%) 2024-09-13T23:34:25.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:25 smithi022 ceph-mon[80039]: Health check update: Degraded data redundancy: 56/627 objects degraded (8.931%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T23:34:25.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:25 smithi022 ceph-mon[82397]: pgmap v232: 161 pgs: 4 activating, 10 peering, 21 active+undersized, 15 active+undersized+degraded, 111 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/627 objects degraded (8.931%) 2024-09-13T23:34:25.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:25 smithi022 ceph-mon[82397]: Health check update: Degraded data redundancy: 56/627 objects degraded (8.931%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T23:34:25.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:25 smithi076 ceph-mon[63913]: pgmap v232: 161 pgs: 4 activating, 10 peering, 21 active+undersized, 15 active+undersized+degraded, 111 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/627 objects degraded (8.931%) 2024-09-13T23:34:25.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:25 smithi076 ceph-mon[63913]: Health check update: Degraded data redundancy: 56/627 objects degraded (8.931%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T23:34:26.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:26 smithi022 ceph-mon[80039]: from='client.64513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:26.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:26 smithi022 ceph-mon[82397]: from='client.64513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:26.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:26 smithi076 ceph-mon[63913]: from='client.64513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:27.042 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:34:27.615 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:27 smithi022 ceph-mon[80039]: pgmap v233: 161 pgs: 4 activating, 10 peering, 147 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 790 B/s rd, 0 op/s 2024-09-13T23:34:27.615 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:27 smithi022 ceph-mon[80039]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/627 objects degraded (8.931%), 15 pgs degraded) 2024-09-13T23:34:27.615 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:27 smithi022 ceph-mon[80039]: Cluster is now healthy 2024-09-13T23:34:27.616 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:27 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:27.616 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:27 smithi022 ceph-mon[82397]: pgmap v233: 161 pgs: 4 activating, 10 peering, 147 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 790 B/s rd, 0 op/s 2024-09-13T23:34:27.616 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:27 smithi022 ceph-mon[82397]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/627 objects degraded (8.931%), 15 pgs degraded) 2024-09-13T23:34:27.616 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:27 smithi022 ceph-mon[82397]: Cluster is now healthy 2024-09-13T23:34:27.616 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:27 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:27.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:27 smithi076 ceph-mon[63913]: pgmap v233: 161 pgs: 4 activating, 10 peering, 147 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 790 B/s rd, 0 op/s 2024-09-13T23:34:27.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:27 smithi076 ceph-mon[63913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/627 objects degraded (8.931%), 15 pgs degraded) 2024-09-13T23:34:27.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:27 smithi076 ceph-mon[63913]: Cluster is now healthy 2024-09-13T23:34:27.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:27 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:28.661 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:28 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:28.661 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:28 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:28.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:28 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:29.423 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:34:29.423 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (21m) 2m ago 29m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:34:29.423 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (8m) 11s ago 29m 83.9M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:34:29.423 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (8m) 2m ago 28m 58.7M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:34:29.424 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (8m) 11s ago 34m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:34:29.424 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (18m) 2m ago 35m 503M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:34:29.424 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (6m) 2m ago 36m 53.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:34:29.424 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (7m) 11s ago 34m 48.5M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:34:29.424 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (6m) 2m ago 34m 42.5M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:34:29.424 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (21m) 2m ago 30m 16.1M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:34:29.424 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (21m) 11s ago 30m 15.7M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:34:29.424 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (4m) 2m ago 33m 51.8M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:34:29.424 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (4m) 2m ago 33m 52.8M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:34:29.424 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (3m) 2m ago 33m 48.6M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 42db9ab27cb1 2024-09-13T23:34:29.424 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (2m) 2m ago 32m 14.5M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 b891533f55d3 2024-09-13T23:34:29.424 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (116s) 11s ago 32m 53.8M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 ba143ef88b4e 2024-09-13T23:34:29.425 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (82s) 11s ago 31m 53.4M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 57c6e1a1827d 2024-09-13T23:34:29.425 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (48s) 11s ago 31m 46.6M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 c0ab477f2b96 2024-09-13T23:34:29.425 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (15s) 11s ago 31m 12.1M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 4325dc4e4077 2024-09-13T23:34:29.425 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (8m) 11s ago 30m 69.5M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:34:29.425 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (28m) 2m ago 28m 78.5M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:34:29.425 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (28m) 11s ago 28m 78.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:34:29.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:29 smithi022 ceph-mon[80039]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:29.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:29 smithi022 ceph-mon[82397]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:29.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:29 smithi076 ceph-mon[63913]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:30.134 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-09-13T23:34:30.655 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:30 smithi022 ceph-mon[80039]: from='client.54582 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:30.655 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:30 smithi022 ceph-mon[82397]: from='client.54582 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:30.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:30 smithi076 ceph-mon[63913]: from='client.54582 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:31.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:31 smithi022 ceph-mon[80039]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T23:34:31.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:31 smithi022 ceph-mon[82397]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T23:34:31.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:31 smithi076 ceph-mon[63913]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T23:34:32.304 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:34:32.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:32 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/1396092461' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:32.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:34:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:34:32] "GET /metrics HTTP/1.1" 200 38164 "" "Prometheus/2.43.0" 2024-09-13T23:34:32.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:32 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/1396092461' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:32.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:32 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/1396092461' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:32.983 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-09-13T23:34:33.611 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:33 smithi022 ceph-mon[80039]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:34:33.611 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:33 smithi022 ceph-mon[82397]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:34:33.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:33 smithi076 ceph-mon[63913]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:34:35.198 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)" 2024-09-13T23:34:35.836 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T23:34:36.139 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:35 smithi022 ceph-mon[82397]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:36.140 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:35 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/2470999666' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:36.140 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:35 smithi022 ceph-mon[80039]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:36.140 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:35 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/2470999666' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:35 smithi076 ceph-mon[63913]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:35 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/2470999666' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:37.345 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:37.345 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:34:37.345 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:37 smithi022 ceph-mon[80039]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:37.345 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:37.346 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:37.346 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:34:37.346 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:37 smithi022 ceph-mon[82397]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:37.346 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:37.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:37.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:34:37.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:37 smithi076 ceph-mon[63913]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:37.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:37.809 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:34:37.809 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": null, 2024-09-13T23:34:37.810 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": false, 2024-09-13T23:34:37.810 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "", 2024-09-13T23:34:37.810 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:34:37.810 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": null, 2024-09-13T23:34:37.810 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-09-13T23:34:37.810 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:34:37.810 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:34:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:38 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:38.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:38 smithi022 ceph-mon[80039]: from='client.64537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:38.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:38 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:38.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:38 smithi022 ceph-mon[82397]: from='client.64537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:38.470 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T23:34:38.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:38 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:38.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:38 smithi076 ceph-mon[63913]: from='client.64537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:39.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:39 smithi022 ceph-mon[80039]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:39.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:39 smithi022 ceph-mon[82397]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:39 smithi076 ceph-mon[63913]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:40.592 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-09-13T23:34:41.324 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-09-13T23:34:41.579 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:41 smithi022 ceph-mon[82397]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:41.579 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:41 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/361629261' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:34:41.579 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:41 smithi022 ceph-mon[80039]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:41.580 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:41 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/361629261' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:34:41.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:41 smithi076 ceph-mon[63913]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:41 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/361629261' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:34:42.573 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:34:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:34:42] "GET /metrics HTTP/1.1" 200 38316 "" "Prometheus/2.43.0" 2024-09-13T23:34:43.409 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:43 smithi022 ceph-mon[80039]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:43.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:43 smithi022 ceph-mon[82397]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:43.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:43 smithi076 ceph-mon[63913]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:44.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:44 smithi022 ceph-mon[80039]: from='client.64549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:44.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:44 smithi022 ceph-mon[82397]: from='client.64549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:44.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:44 smithi076 ceph-mon[63913]: from='client.64549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:44.807 INFO:teuthology.orchestra.run.smithi022.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:34:45.446 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T23:34:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[80039]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[80039]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:34:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:34:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[80039]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:34:46.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[82397]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:46.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[82397]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:34:46.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:46.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:46.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:34:46.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:46.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:45 smithi022 ceph-mon[82397]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:34:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:45 smithi076 ceph-mon[63913]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:45 smithi076 ceph-mon[63913]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:34:46.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:46.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:46.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:34:46.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:45 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:46.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:45 smithi076 ceph-mon[63913]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:34:47.728 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:47.728 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:47.728 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:34:47.728 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:34:47.728 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:34:47.728 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mgr 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mon 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all crash 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all osd 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mds 2024-09-13T23:34:47.729 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:47.730 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:47.730 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:47.730 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:34:47.730 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:34:47.730 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:34:47.730 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:47.730 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mgr 2024-09-13T23:34:47.730 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:47.730 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:47.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mon 2024-09-13T23:34:47.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:47.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:47.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all crash 2024-09-13T23:34:47.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:47.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:47.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all osd 2024-09-13T23:34:47.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:47.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:47.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mds 2024-09-13T23:34:47.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:47 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:48.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:48.040 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:48.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:34:48.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:34:48.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:34:48.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:48.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mgr 2024-09-13T23:34:48.041 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:48.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:48.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mon 2024-09-13T23:34:48.042 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:48.043 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:48.043 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all crash 2024-09-13T23:34:48.043 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:48.044 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:48.044 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all osd 2024-09-13T23:34:48.044 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:48.044 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:48.044 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mds 2024-09-13T23:34:48.044 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:47 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:48.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:48 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:48.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:48 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:49.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:48 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:49.143 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:34:49.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:49 smithi076 ceph-mon[63913]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:49.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:49 smithi076 ceph-mon[63913]: from='client.54618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:49.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:49.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi022.snmrkm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:34:49.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:49 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:49.844 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:34:49.845 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (21m) 2m ago 29m 24.5M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:34:49.845 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (8m) 31s ago 29m 83.9M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:34:49.845 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (9m) 2m ago 29m 58.7M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:34:49.845 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (8m) 31s ago 34m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:34:49.845 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (18m) 2m ago 36m 503M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:34:49.845 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (6m) 2m ago 36m 53.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:34:49.845 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (7m) 31s ago 34m 48.5M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:34:49.845 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (6m) 2m ago 35m 42.5M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:34:49.845 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (22m) 2m ago 30m 16.1M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:34:49.845 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (21m) 31s ago 30m 15.7M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:34:49.845 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (5m) 2m ago 34m 51.8M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:34:49.846 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (4m) 2m ago 33m 52.8M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:34:49.846 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (3m) 2m ago 33m 48.6M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 42db9ab27cb1 2024-09-13T23:34:49.846 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (2m) 2m ago 33m 14.5M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 b891533f55d3 2024-09-13T23:34:49.846 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (2m) 31s ago 32m 53.8M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 ba143ef88b4e 2024-09-13T23:34:49.846 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (103s) 31s ago 32m 53.4M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 57c6e1a1827d 2024-09-13T23:34:49.846 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (69s) 31s ago 31m 46.6M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 c0ab477f2b96 2024-09-13T23:34:49.846 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (36s) 31s ago 31m 12.1M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 4325dc4e4077 2024-09-13T23:34:49.846 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (9m) 31s ago 30m 69.5M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:34:49.846 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (29m) 2m ago 29m 78.5M - 17.2.0 e1d6a67b021e 0b354b5870dd 2024-09-13T23:34:49.846 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (29m) 31s ago 29m 78.0M - 17.2.0 e1d6a67b021e 1b5729613739 2024-09-13T23:34:49.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:49 smithi022 ceph-mon[80039]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:49.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:49 smithi022 ceph-mon[80039]: from='client.54618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:49.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:49.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi022.snmrkm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:34:49.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:49 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:49.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:49 smithi022 ceph-mon[82397]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:49.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:49 smithi022 ceph-mon[82397]: from='client.54618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:49.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:49.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi022.snmrkm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:34:49.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:49 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:50.266 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:34:50.267 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:34:50.267 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:34:50.267 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:34:50.267 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:34:50.267 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:34:50.267 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:34:50.267 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:34:50.267 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T23:34:50.267 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:34:50.267 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:34:50.267 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T23:34:50.267 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:34:50.268 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:34:50.268 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-13T23:34:50.268 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 13 2024-09-13T23:34:50.268 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:34:50.268 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:34:50.620 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:34:50.621 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:34:50.621 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:34:50.621 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-09-13T23:34:50.621 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:34:50.621 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "0/2 daemons upgraded", 2024-09-13T23:34:50.621 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading rgw daemons", 2024-09-13T23:34:50.621 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:34:50.621 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:34:50.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:50 smithi022 ceph-mon[80039]: Upgrade: Updating rgw.foo.smithi022.snmrkm (1/2) 2024-09-13T23:34:50.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:50 smithi022 ceph-mon[80039]: Deploying daemon rgw.foo.smithi022.snmrkm on smithi022 2024-09-13T23:34:50.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:50 smithi022 ceph-mon[80039]: from='client.64561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:50.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:50 smithi022 ceph-mon[80039]: from='client.54630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:50.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:50 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/1490757830' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:50.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:50 smithi022 ceph-mon[82397]: Upgrade: Updating rgw.foo.smithi022.snmrkm (1/2) 2024-09-13T23:34:50.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:50 smithi022 ceph-mon[82397]: Deploying daemon rgw.foo.smithi022.snmrkm on smithi022 2024-09-13T23:34:50.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:50 smithi022 ceph-mon[82397]: from='client.64561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:50.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:50 smithi022 ceph-mon[82397]: from='client.54630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:50.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:50 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/1490757830' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:51.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:50 smithi076 ceph-mon[63913]: Upgrade: Updating rgw.foo.smithi022.snmrkm (1/2) 2024-09-13T23:34:51.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:50 smithi076 ceph-mon[63913]: Deploying daemon rgw.foo.smithi022.snmrkm on smithi022 2024-09-13T23:34:51.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:50 smithi076 ceph-mon[63913]: from='client.64561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:51.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:50 smithi076 ceph-mon[63913]: from='client.54630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:51.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:50 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/1490757830' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:34:51.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:51 smithi022 ceph-mon[80039]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:51.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:51 smithi022 ceph-mon[80039]: from='client.64579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:51.929 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:51 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:34:51.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:51 smithi022 ceph-mon[82397]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:51.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:51 smithi022 ceph-mon[82397]: from='client.64579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:51.929 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:51 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:34:52.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:51 smithi076 ceph-mon[63913]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:52.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:51 smithi076 ceph-mon[63913]: from='client.64579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:34:52.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:51 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:34:52.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:34:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:34:52] "GET /metrics HTTP/1.1" 200 38319 "" "Prometheus/2.43.0" 2024-09-13T23:34:53.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:53 smithi022 ceph-mon[82397]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:53.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:53 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:53.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:53 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:53 smithi022 ceph-mon[80039]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:53 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:53.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:53 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:53.945 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:53 smithi076 ceph-mon[63913]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:34:53.945 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:53 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:53.945 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:53 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:55.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:55 smithi076 ceph-mon[63913]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:55.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:55 smithi076 ceph-mon[63913]: Upgrade: Updating rgw.foo.smithi076.rnamrn (2/2) 2024-09-13T23:34:55.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:55 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:55.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:55 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi076.rnamrn", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:34:55.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:55 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:55.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:55 smithi076 ceph-mon[63913]: Deploying daemon rgw.foo.smithi076.rnamrn on smithi076 2024-09-13T23:34:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:55 smithi022 ceph-mon[80039]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:55 smithi022 ceph-mon[80039]: Upgrade: Updating rgw.foo.smithi076.rnamrn (2/2) 2024-09-13T23:34:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:55 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:55 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi076.rnamrn", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:34:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:55 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:55 smithi022 ceph-mon[80039]: Deploying daemon rgw.foo.smithi076.rnamrn on smithi076 2024-09-13T23:34:56.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:55 smithi022 ceph-mon[82397]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:34:56.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:55 smithi022 ceph-mon[82397]: Upgrade: Updating rgw.foo.smithi076.rnamrn (2/2) 2024-09-13T23:34:56.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:55 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:56.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:55 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi076.rnamrn", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T23:34:56.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:55 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:34:56.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:55 smithi022 ceph-mon[82397]: Deploying daemon rgw.foo.smithi076.rnamrn on smithi076 2024-09-13T23:34:57.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:57 smithi022 ceph-mon[82397]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 39 op/s 2024-09-13T23:34:57.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:57 smithi022 ceph-mon[80039]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 39 op/s 2024-09-13T23:34:58.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:57 smithi076 ceph-mon[63913]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 39 op/s 2024-09-13T23:34:59.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:58 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:59.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:58 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:59.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:58 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:59.092 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:58 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:59.092 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:58 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:59.093 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:58 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:59.093 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:58 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:34:59.093 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:58 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:34:59.093 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:58 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:00.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:34:59 smithi076 ceph-mon[63913]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-09-13T23:35:00.142 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:34:59 smithi022 ceph-mon[80039]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-09-13T23:35:00.142 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:34:59 smithi022 ceph-mon[82397]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-09-13T23:35:01.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:01 smithi022 ceph-mon[80039]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 85 B/s wr, 111 op/s 2024-09-13T23:35:01.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:01 smithi022 ceph-mon[82397]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 85 B/s wr, 111 op/s 2024-09-13T23:35:02.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:01 smithi076 ceph-mon[63913]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 85 B/s wr, 111 op/s 2024-09-13T23:35:02.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:35:02] "GET /metrics HTTP/1.1" 200 38319 "" "Prometheus/2.43.0" 2024-09-13T23:35:03.233 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:02 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.233 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:02 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.233 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:02 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.233 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:02 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.233 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:02 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.233 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:02 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:03.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:02 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:04.099 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:03 smithi022 ceph-mon[82397]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 170 B/s wr, 141 op/s 2024-09-13T23:35:04.099 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:03 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:04.100 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:03 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:04.100 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:03 smithi022 ceph-mon[80039]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 170 B/s wr, 141 op/s 2024-09-13T23:35:04.100 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:03 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:04.100 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:03 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:03 smithi076 ceph-mon[63913]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 170 B/s wr, 141 op/s 2024-09-13T23:35:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:03 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:03 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:06.268 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:05 smithi022 ceph-mon[80039]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 168 op/s 2024-09-13T23:35:06.268 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:05 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:06.268 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:05 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:06.269 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:05 smithi022 ceph-mon[82397]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 168 op/s 2024-09-13T23:35:06.269 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:05 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:06.269 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:05 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:05 smithi076 ceph-mon[63913]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 168 op/s 2024-09-13T23:35:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:05 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:05 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.185 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:35:07.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:07.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:07.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:35:07.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.186 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi022.snmrkm"}]: dispatch 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi022.snmrkm"}]': finished 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi076.rnamrn"}]: dispatch 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi076.rnamrn"}]': finished 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.188 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.188 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.188 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.188 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:35:07.188 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.188 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.188 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:07.188 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi022.snmrkm"}]: dispatch 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi022.snmrkm"}]': finished 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi076.rnamrn"}]: dispatch 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi076.rnamrn"}]': finished 2024-09-13T23:35:07.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.190 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.190 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.190 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.190 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.190 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.190 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.190 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.190 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.190 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.190 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.191 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.191 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.191 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.191 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.191 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:35:07.191 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:35:07.191 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.191 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:35:07.191 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:35:07.191 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:35:07.191 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:35:07.191 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T23:35:07.192 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:35:07.192 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:35:07.192 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:35:07.192 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T23:35:07.192 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:35:07.192 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:35:07.192 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.192 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:35:07.192 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:35:07.192 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:35:07.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:35:07.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:35:07.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:35:07.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:35:07.193 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:35:07.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:35:07.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T23:35:07.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:35:07.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:35:07.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:35:07.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T23:35:07.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:35:07.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:35:07.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:35:07.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:35:07.194 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:35:07.195 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:35:07.195 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:35:07.195 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:35:07.195 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:35:07.195 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.195 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.195 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.195 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.195 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.195 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.196 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:35:07.196 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:35:07.196 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:07.196 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:07.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:35:07.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:35:07.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:35:07.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:35:07.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:35:07.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:35:07.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:07.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:07.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:35:07.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:07.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:07.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:35:07.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi022.snmrkm"}]: dispatch 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi022.snmrkm"}]': finished 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi076.rnamrn"}]: dispatch 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi076.rnamrn"}]': finished 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.289 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.290 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T23:35:07.291 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T23:35:07.292 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T23:35:07.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:07.293 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:08.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-09-13T23:35:08.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all rgw 2024-09-13T23:35:08.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:35:08.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:35:08.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all nfs 2024-09-13T23:35:08.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:35:08.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: Upgrade: Finalizing container_image settings 2024-09-13T23:35:08.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: Upgrade: Complete! 2024-09-13T23:35:08.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: Checking dashboard <-> RGW credentials 2024-09-13T23:35:08.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:08.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:08.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:08.428 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:08.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-09-13T23:35:08.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all rgw 2024-09-13T23:35:08.428 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:35:08.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:35:08.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all nfs 2024-09-13T23:35:08.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:35:08.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: Upgrade: Finalizing container_image settings 2024-09-13T23:35:08.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: Upgrade: Complete! 2024-09-13T23:35:08.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: Checking dashboard <-> RGW credentials 2024-09-13T23:35:08.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:08.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:08.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:08.429 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:08 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:08.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-09-13T23:35:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all rgw 2024-09-13T23:35:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:35:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:35:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all nfs 2024-09-13T23:35:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all nvmeof 2024-09-13T23:35:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: Upgrade: Finalizing container_image settings 2024-09-13T23:35:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: Upgrade: Complete! 2024-09-13T23:35:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: Checking dashboard <-> RGW credentials 2024-09-13T23:35:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:08.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:08.539 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:08 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:09.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:09 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:09.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:09 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:09.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:09 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:10.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:10 smithi076 ceph-mon[63913]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 170 B/s wr, 129 op/s 2024-09-13T23:35:10.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:10 smithi022 ceph-mon[80039]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 170 B/s wr, 129 op/s 2024-09-13T23:35:10.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:10 smithi022 ceph-mon[82397]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 170 B/s wr, 129 op/s 2024-09-13T23:35:12.452 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:12 smithi022 ceph-mon[82397]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 85 B/s wr, 85 op/s 2024-09-13T23:35:12.452 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:12 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:12.452 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:12 smithi022 ceph-mon[80039]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 85 B/s wr, 85 op/s 2024-09-13T23:35:12.453 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:12 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:12 smithi076 ceph-mon[63913]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 85 B/s wr, 85 op/s 2024-09-13T23:35:12.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:12 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:12.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:35:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:35:12] "GET /metrics HTTP/1.1" 200 38348 "" "Prometheus/2.43.0" 2024-09-13T23:35:14.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:14 smithi076 ceph-mon[63913]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 85 B/s wr, 59 op/s 2024-09-13T23:35:14.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:14 smithi022 ceph-mon[80039]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 85 B/s wr, 59 op/s 2024-09-13T23:35:14.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:14 smithi022 ceph-mon[82397]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 85 B/s wr, 59 op/s 2024-09-13T23:35:16.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:16 smithi076 ceph-mon[63913]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 29 op/s 2024-09-13T23:35:16.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:16 smithi022 ceph-mon[82397]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 29 op/s 2024-09-13T23:35:16.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:16 smithi022 ceph-mon[80039]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 29 op/s 2024-09-13T23:35:18.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:18 smithi022 ceph-mon[80039]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-09-13T23:35:18.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:18 smithi022 ceph-mon[82397]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-09-13T23:35:18.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:18 smithi076 ceph-mon[63913]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-09-13T23:35:19.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:19 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:19.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:19 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:19 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:20.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:20 smithi076 ceph-mon[63913]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-09-13T23:35:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:20 smithi022 ceph-mon[80039]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-09-13T23:35:20.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:20 smithi022 ceph-mon[82397]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-09-13T23:35:21.779 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:35:22.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:22 smithi022 ceph-mon[80039]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-09-13T23:35:22.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:22 smithi022 ceph-mon[80039]: from='client.64660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:22.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:22 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:35:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:22 smithi022 ceph-mon[82397]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-09-13T23:35:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:22 smithi022 ceph-mon[82397]: from='client.64660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:22.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:22 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:35:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:22 smithi076 ceph-mon[63913]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-09-13T23:35:22.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:22 smithi076 ceph-mon[63913]: from='client.64660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:22.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:22 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:35:22.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:35:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:35:22] "GET /metrics HTTP/1.1" 200 38347 "" "Prometheus/2.43.0" 2024-09-13T23:35:23.782 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:35:23.782 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (22m) 21s ago 30m 24.8M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (9m) 21s ago 30m 82.2M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (9m) 21s ago 29m 61.0M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (9m) 21s ago 35m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (19m) 21s ago 36m 508M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (7m) 21s ago 36m 61.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (8m) 21s ago 35m 51.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (7m) 21s ago 35m 49.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (22m) 21s ago 31m 15.6M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (22m) 21s ago 31m 15.8M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (5m) 21s ago 34m 59.2M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (5m) 21s ago 34m 61.2M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (4m) 21s ago 34m 55.3M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 42db9ab27cb1 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (3m) 21s ago 33m 53.5M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 b891533f55d3 2024-09-13T23:35:23.783 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (2m) 21s ago 33m 57.6M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 ba143ef88b4e 2024-09-13T23:35:23.784 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (2m) 21s ago 32m 55.4M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 57c6e1a1827d 2024-09-13T23:35:23.784 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (103s) 21s ago 32m 49.6M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 c0ab477f2b96 2024-09-13T23:35:23.784 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (70s) 21s ago 32m 54.5M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 4325dc4e4077 2024-09-13T23:35:23.784 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (9m) 21s ago 31m 69.6M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:35:23.784 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (30s) 21s ago 29m 77.8M - 19.1.1-324-gecab7be5 0652ff2b5d78 eb4e10d75d7f 2024-09-13T23:35:23.784 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (25s) 21s ago 29m 75.0M - 19.1.1-324-gecab7be5 0652ff2b5d78 833b94cee8f5 2024-09-13T23:35:24.379 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:24 smithi022 ceph-mon[80039]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T23:35:24.379 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:24 smithi022 ceph-mon[82397]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T23:35:24.407 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-09-13T23:35:24.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:24 smithi076 ceph-mon[63913]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T23:35:25.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:25 smithi022 ceph-mon[80039]: from='client.54732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:25.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:25 smithi022 ceph-mon[82397]: from='client.54732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:25.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:25 smithi076 ceph-mon[63913]: from='client.54732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:26.432 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:35:26.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:26 smithi076 ceph-mon[63913]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:26.676 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:26 smithi022 ceph-mon[82397]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:26.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:26 smithi022 ceph-mon[80039]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:27.188 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-09-13T23:35:27.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:27 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/1866796337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:27.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:27 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/1866796337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:27.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:27 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/1866796337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:28.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:28 smithi022 ceph-mon[80039]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:28.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:28 smithi022 ceph-mon[82397]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:28.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:28 smithi076 ceph-mon[63913]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:29.361 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)" 2024-09-13T23:35:29.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:29 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:29.647 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:29 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:29.648 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:29 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:30.048 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T23:35:30.355 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:30 smithi022 ceph-mon[80039]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:30.355 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:30 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/4221741942' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:30.355 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:30 smithi022 ceph-mon[82397]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:30.355 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:30 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/4221741942' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:30 smithi076 ceph-mon[63913]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:30.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:30 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/4221741942' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:32.321 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:35:32.321 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": null, 2024-09-13T23:35:32.321 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": false, 2024-09-13T23:35:32.321 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "", 2024-09-13T23:35:32.321 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-13T23:35:32.322 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": null, 2024-09-13T23:35:32.322 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-09-13T23:35:32.322 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:35:32.322 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:35:32.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:32 smithi076 ceph-mon[63913]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:35:32.629 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:32 smithi022 ceph-mon[82397]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:35:32.630 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:35:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:35:32] "GET /metrics HTTP/1.1" 200 38347 "" "Prometheus/2.43.0" 2024-09-13T23:35:32.630 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:32 smithi022 ceph-mon[80039]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:35:33.014 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T23:35:34.493 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:34 smithi022 ceph-mon[80039]: from='client.64684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:34.493 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:34 smithi022 ceph-mon[80039]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:35:34.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:34 smithi022 ceph-mon[82397]: from='client.64684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:34.493 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:34 smithi022 ceph-mon[82397]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:35:34.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:34 smithi076 ceph-mon[63913]: from='client.64684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:34.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:34 smithi076 ceph-mon[63913]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:35:35.140 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-09-13T23:35:35.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:35 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/1603162744' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:35:35.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:35 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/1603162744' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:35:35.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:35 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/1603162744' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:35:35.854 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T23:35:36.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:36 smithi022 ceph-mon[80039]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:36.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:36 smithi022 ceph-mon[82397]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:36.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:36 smithi076 ceph-mon[63913]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:37.420 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:37 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:35:37.420 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:37 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:35:37.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:37 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:35:38.048 INFO:teuthology.orchestra.run.smithi022.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:35:38.334 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:38 smithi022 ceph-mon[80039]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:35:38.334 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:38.335 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:38.335 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:38.335 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:38 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:38.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:38 smithi022 ceph-mon[82397]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:35:38.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:38.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:38.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:38.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:38 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:38 smithi076 ceph-mon[63913]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T23:35:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:38.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:38.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:38.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:38 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:38.801 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T23:35:38.812 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi022.front.sepia.ceph.com 2024-09-13T23:35:38.812 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-13T23:35:39.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:39 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:39.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:39 smithi022 ceph-mon[80039]: from='client.64696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:39.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:39 smithi022 ceph-mon[80039]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:35:39.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:39 smithi022 ceph-mon[80039]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:35:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:39 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:39 smithi022 ceph-mon[82397]: from='client.64696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:39 smithi022 ceph-mon[82397]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:35:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:39 smithi022 ceph-mon[82397]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:35:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:39 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:39 smithi076 ceph-mon[63913]: from='client.64696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:39.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:39 smithi076 ceph-mon[63913]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:35:39.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:39 smithi076 ceph-mon[63913]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:35:40.511 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:40 smithi022 ceph-mon[80039]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:40.511 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:40 smithi022 ceph-mon[82397]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:40.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:40 smithi076 ceph-mon[63913]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:41.770 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:41.770 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.770 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:35:41.770 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:35:41.770 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mgr 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mon 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all crash 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all osd 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all mds 2024-09-13T23:35:41.772 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.772 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.772 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all rgw 2024-09-13T23:35:41.772 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.772 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.772 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:35:41.772 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.772 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.772 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T23:35:41.772 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.772 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:35:41.772 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:41.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:35:41.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:35:41.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:35:41.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mgr 2024-09-13T23:35:41.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mon 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all crash 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all osd 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all mds 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all rgw 2024-09-13T23:35:41.774 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.775 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.775 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:35:41.775 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.775 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.775 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T23:35:41.775 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.775 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:35:41.775 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:41 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:41.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T23:35:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T23:35:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T23:35:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mgr 2024-09-13T23:35:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mon 2024-09-13T23:35:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all crash 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all osd 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all mds 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all rgw 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T23:35:41.789 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T23:35:41.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:41.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T23:35:41.790 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:41 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:41.868 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:35:42.559 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:35:42.559 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (22m) 39s ago 30m 24.8M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:35:42.559 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (9m) 40s ago 30m 82.2M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:35:42.559 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (9m) 39s ago 29m 61.0M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:35:42.559 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (9m) 40s ago 35m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:35:42.559 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (19m) 39s ago 37m 508M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:35:42.559 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (7m) 39s ago 37m 61.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:35:42.559 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (8m) 40s ago 35m 51.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (7m) 39s ago 35m 49.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (22m) 39s ago 31m 15.6M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (22m) 40s ago 31m 15.8M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (6m) 39s ago 35m 59.2M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (5m) 39s ago 34m 61.2M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (4m) 39s ago 34m 55.3M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 42db9ab27cb1 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (3m) 39s ago 33m 53.5M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 b891533f55d3 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (3m) 40s ago 33m 57.6M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 ba143ef88b4e 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (2m) 40s ago 33m 55.4M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 57c6e1a1827d 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (2m) 40s ago 32m 49.6M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 c0ab477f2b96 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (88s) 40s ago 32m 54.5M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 4325dc4e4077 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (9m) 40s ago 31m 69.6M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:35:42.560 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (49s) 39s ago 30m 77.8M - 19.1.1-324-gecab7be5 0652ff2b5d78 eb4e10d75d7f 2024-09-13T23:35:42.561 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (44s) 40s ago 30m 75.0M - 19.1.1-324-gecab7be5 0652ff2b5d78 833b94cee8f5 2024-09-13T23:35:42.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:35:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:35:42] "GET /metrics HTTP/1.1" 200 38348 "" "Prometheus/2.43.0" 2024-09-13T23:35:42.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:42 smithi022 ceph-mon[80039]: from='client.64702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:42.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:42 smithi022 ceph-mon[80039]: from='client.64708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:42.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:42 smithi022 ceph-mon[82397]: from='client.64702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:42.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:42 smithi022 ceph-mon[82397]: from='client.64708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:42.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:42 smithi076 ceph-mon[63913]: from='client.64702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:42.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:42 smithi076 ceph-mon[63913]: from='client.64708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:42.982 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:35:42.982 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:35:42.982 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:35:42.982 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:35:42.982 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:35:42.982 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:35:42.982 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:35:42.982 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:35:42.982 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T23:35:42.982 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:35:42.983 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:35:42.983 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:35:42.983 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:35:42.983 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:35:42.983 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 15 2024-09-13T23:35:42.983 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:35:42.983 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:35:43.350 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:35:43.350 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:35:43.350 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:35:43.350 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T23:35:43.351 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-13T23:35:43.351 INFO:teuthology.orchestra.run.smithi022.stdout: "osd", 2024-09-13T23:35:43.351 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr", 2024-09-13T23:35:43.351 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw", 2024-09-13T23:35:43.351 INFO:teuthology.orchestra.run.smithi022.stdout: "mon" 2024-09-13T23:35:43.351 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-13T23:35:43.352 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "15/21 daemons upgraded", 2024-09-13T23:35:43.352 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading iscsi daemons", 2024-09-13T23:35:43.352 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-13T23:35:43.352 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:35:43.769 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_OK 2024-09-13T23:35:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[82397]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[82397]: from='client.54780 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[82397]: Upgrade: Updating iscsi.foo.smithi022.wguwxm 2024-09-13T23:35:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:35:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[82397]: Deploying daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:35:44.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/596588357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:44.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/1446616214' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:35:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[80039]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[80039]: from='client.54780 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[80039]: Upgrade: Updating iscsi.foo.smithi022.wguwxm 2024-09-13T23:35:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:35:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[80039]: Deploying daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:35:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/596588357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:44.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:43 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/1446616214' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:35:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:43 smithi076 ceph-mon[63913]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:44.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:43 smithi076 ceph-mon[63913]: from='client.54780 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:44.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:43 smithi076 ceph-mon[63913]: Upgrade: Updating iscsi.foo.smithi022.wguwxm 2024-09-13T23:35:44.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:43 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:44.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:43 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi022.wguwxm", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T23:35:44.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:43 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:44.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:43 smithi076 ceph-mon[63913]: Deploying daemon iscsi.foo.smithi022.wguwxm on smithi022 2024-09-13T23:35:44.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:43 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/596588357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:35:44.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:43 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/1446616214' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:35:45.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:44 smithi022 ceph-mon[80039]: from='client.64726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:45.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:44 smithi022 ceph-mon[82397]: from='client.64726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:44 smithi076 ceph-mon[63913]: from='client.64726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:35:46.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:45 smithi022 ceph-mon[80039]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:46.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:45 smithi022 ceph-mon[82397]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:45 smithi076 ceph-mon[63913]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:48.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:47 smithi022 ceph-mon[80039]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:48.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:47 smithi022 ceph-mon[82397]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:48.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:47 smithi076 ceph-mon[63913]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:49.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:48 smithi022 ceph-mon[80039]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:49.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:48 smithi022 ceph-mon[82397]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:49.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:48 smithi076 ceph-mon[63913]: from='client.15558 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:35:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:49 smithi022 ceph-mon[80039]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:50.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:49 smithi022 ceph-mon[82397]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:49 smithi076 ceph-mon[63913]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:52.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:51 smithi022 ceph-mon[80039]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:52.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:51 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:35:52.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:51 smithi022 ceph-mon[82397]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:52.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:51 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:35:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:51 smithi076 ceph-mon[63913]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:51 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:35:52.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:35:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:35:52] "GET /metrics HTTP/1.1" 200 38347 "" "Prometheus/2.43.0" 2024-09-13T23:35:54.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:53 smithi022 ceph-mon[80039]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:54.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:53 smithi022 ceph-mon[82397]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:54.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:53 smithi076 ceph-mon[63913]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:56.124 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:55 smithi022 ceph-mon[82397]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:56.125 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:55 smithi022 ceph-mon[80039]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:55 smithi076 ceph-mon[63913]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:35:56.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:35:56 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:35:56.422+0000 7fd7c80f8640 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi022.wguwxm on host smithi022 failed. 2024-09-13T23:35:57.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:57 smithi076 ceph-mon[63913]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:57.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:57 smithi076 ceph-mon[63913]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi022.wguwxm on host smithi022 failed. 2024-09-13T23:35:57.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:57.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:57 smithi076 ceph-mon[63913]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1018 B/s rd, 0 op/s 2024-09-13T23:35:57.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:57.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:57.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:57.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:57 smithi076 ceph-mon[63913]: Health check failed: Upgrading daemon iscsi.foo.smithi022.wguwxm on host smithi022 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-09-13T23:35:57.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[80039]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:57.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[80039]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi022.wguwxm on host smithi022 failed. 2024-09-13T23:35:57.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:57.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[80039]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1018 B/s rd, 0 op/s 2024-09-13T23:35:57.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:57.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:57.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:57.928 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[80039]: Health check failed: Upgrading daemon iscsi.foo.smithi022.wguwxm on host smithi022 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-09-13T23:35:57.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[82397]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:35:57.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[82397]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi022.wguwxm on host smithi022 failed. 2024-09-13T23:35:57.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:57.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[82397]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1018 B/s rd, 0 op/s 2024-09-13T23:35:57.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:35:57.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:35:57.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:35:57.928 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:57 smithi022 ceph-mon[82397]: Health check failed: Upgrading daemon iscsi.foo.smithi022.wguwxm on host smithi022 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-09-13T23:35:59.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:35:59 smithi076 ceph-mon[63913]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 814 B/s rd, 0 op/s 2024-09-13T23:35:59.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:35:59 smithi022 ceph-mon[80039]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 814 B/s rd, 0 op/s 2024-09-13T23:35:59.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:35:59 smithi022 ceph-mon[82397]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 814 B/s rd, 0 op/s 2024-09-13T23:36:01.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:01 smithi076 ceph-mon[63913]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 814 B/s rd, 0 op/s 2024-09-13T23:36:01.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:01 smithi022 ceph-mon[80039]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 814 B/s rd, 0 op/s 2024-09-13T23:36:01.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:01 smithi022 ceph-mon[82397]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 814 B/s rd, 0 op/s 2024-09-13T23:36:02.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:36:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:36:02] "GET /metrics HTTP/1.1" 200 38347 "" "Prometheus/2.43.0" 2024-09-13T23:36:03.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:03 smithi076 ceph-mon[63913]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 814 B/s rd, 0 op/s 2024-09-13T23:36:03.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:03 smithi022 ceph-mon[80039]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 814 B/s rd, 0 op/s 2024-09-13T23:36:03.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:03 smithi022 ceph-mon[82397]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 814 B/s rd, 0 op/s 2024-09-13T23:36:05.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:05 smithi076 ceph-mon[63913]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 305 B/s rd, 0 op/s 2024-09-13T23:36:05.927 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:05 smithi022 ceph-mon[80039]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 305 B/s rd, 0 op/s 2024-09-13T23:36:05.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:05 smithi022 ceph-mon[82397]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 305 B/s rd, 0 op/s 2024-09-13T23:36:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:36:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:07 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:36:07.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:07 smithi022 ceph-mon[82397]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 305 B/s rd, 0 op/s 2024-09-13T23:36:07.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:36:07.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:07 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:36:07.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:07 smithi022 ceph-mon[80039]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 305 B/s rd, 0 op/s 2024-09-13T23:36:07.590 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:36:07.590 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:07 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:36:07.590 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:07 smithi076 ceph-mon[63913]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 305 B/s rd, 0 op/s 2024-09-13T23:36:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:08 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/1046907911' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:36:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:08 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/1768470739' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/271654139"}]: dispatch 2024-09-13T23:36:08.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:08 smithi022 ceph-mon[80039]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/271654139"}]: dispatch 2024-09-13T23:36:08.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:08 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/1046907911' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:36:08.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:08 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/1768470739' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/271654139"}]: dispatch 2024-09-13T23:36:08.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:08 smithi022 ceph-mon[82397]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/271654139"}]: dispatch 2024-09-13T23:36:08.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:08 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/1046907911' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T23:36:08.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:08 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/1768470739' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/271654139"}]: dispatch 2024-09-13T23:36:08.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:08 smithi076 ceph-mon[63913]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/271654139"}]: dispatch 2024-09-13T23:36:09.580 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:09 smithi022 ceph-mon[80039]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/271654139"}]': finished 2024-09-13T23:36:09.580 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:09 smithi022 ceph-mon[80039]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T23:36:09.580 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:09 smithi022 ceph-mon[80039]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-13T23:36:09.580 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:09 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/304578629' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3281985089"}]: dispatch 2024-09-13T23:36:09.580 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:09 smithi022 ceph-mon[80039]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3281985089"}]: dispatch 2024-09-13T23:36:09.580 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:09 smithi022 ceph-mon[82397]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/271654139"}]': finished 2024-09-13T23:36:09.580 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:09 smithi022 ceph-mon[82397]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T23:36:09.580 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:09 smithi022 ceph-mon[82397]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-13T23:36:09.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:09 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/304578629' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3281985089"}]: dispatch 2024-09-13T23:36:09.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:09 smithi022 ceph-mon[82397]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3281985089"}]: dispatch 2024-09-13T23:36:09.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:09 smithi076 ceph-mon[63913]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/271654139"}]': finished 2024-09-13T23:36:09.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:09 smithi076 ceph-mon[63913]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T23:36:09.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:09 smithi076 ceph-mon[63913]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-13T23:36:09.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:09 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/304578629' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3281985089"}]: dispatch 2024-09-13T23:36:09.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:09 smithi076 ceph-mon[63913]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3281985089"}]: dispatch 2024-09-13T23:36:10.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:10 smithi022 ceph-mon[80039]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3281985089"}]': finished 2024-09-13T23:36:10.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:10 smithi022 ceph-mon[80039]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T23:36:10.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:10 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/2995872735' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3790464015"}]: dispatch 2024-09-13T23:36:10.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:10 smithi022 ceph-mon[80039]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3790464015"}]: dispatch 2024-09-13T23:36:10.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:10 smithi022 ceph-mon[82397]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3281985089"}]': finished 2024-09-13T23:36:10.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:10 smithi022 ceph-mon[82397]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T23:36:10.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:10 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/2995872735' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3790464015"}]: dispatch 2024-09-13T23:36:10.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:10 smithi022 ceph-mon[82397]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3790464015"}]: dispatch 2024-09-13T23:36:10.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:10 smithi076 ceph-mon[63913]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3281985089"}]': finished 2024-09-13T23:36:10.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:10 smithi076 ceph-mon[63913]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T23:36:10.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:10 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/2995872735' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3790464015"}]: dispatch 2024-09-13T23:36:10.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:10 smithi076 ceph-mon[63913]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3790464015"}]: dispatch 2024-09-13T23:36:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:11 smithi022 ceph-mon[80039]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3790464015"}]': finished 2024-09-13T23:36:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:11 smithi022 ceph-mon[80039]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T23:36:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:11 smithi022 ceph-mon[80039]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T23:36:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:11 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/2002158118' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3790464015"}]: dispatch 2024-09-13T23:36:11.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:11 smithi022 ceph-mon[80039]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3790464015"}]: dispatch 2024-09-13T23:36:11.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:11 smithi022 ceph-mon[82397]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3790464015"}]': finished 2024-09-13T23:36:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:11 smithi022 ceph-mon[82397]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T23:36:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:11 smithi022 ceph-mon[82397]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T23:36:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:11 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/2002158118' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3790464015"}]: dispatch 2024-09-13T23:36:11.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:11 smithi022 ceph-mon[82397]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3790464015"}]: dispatch 2024-09-13T23:36:11.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:11 smithi076 ceph-mon[63913]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6800/3790464015"}]': finished 2024-09-13T23:36:11.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:11 smithi076 ceph-mon[63913]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T23:36:11.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:11 smithi076 ceph-mon[63913]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T23:36:11.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:11 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/2002158118' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3790464015"}]: dispatch 2024-09-13T23:36:11.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:11 smithi076 ceph-mon[63913]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3790464015"}]: dispatch 2024-09-13T23:36:12.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:36:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:36:12] "GET /metrics HTTP/1.1" 200 38425 "" "Prometheus/2.43.0" 2024-09-13T23:36:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:12 smithi022 ceph-mon[80039]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3790464015"}]': finished 2024-09-13T23:36:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:12 smithi022 ceph-mon[80039]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T23:36:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:12 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/1027409008' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3511008014"}]: dispatch 2024-09-13T23:36:12.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:12 smithi022 ceph-mon[80039]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3511008014"}]: dispatch 2024-09-13T23:36:12.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:12 smithi022 ceph-mon[82397]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3790464015"}]': finished 2024-09-13T23:36:12.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:12 smithi022 ceph-mon[82397]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T23:36:12.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:12 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/1027409008' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3511008014"}]: dispatch 2024-09-13T23:36:12.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:12 smithi022 ceph-mon[82397]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3511008014"}]: dispatch 2024-09-13T23:36:12.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:12 smithi076 ceph-mon[63913]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:6801/3790464015"}]': finished 2024-09-13T23:36:12.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:12 smithi076 ceph-mon[63913]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T23:36:12.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:12 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/1027409008' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3511008014"}]: dispatch 2024-09-13T23:36:12.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:12 smithi076 ceph-mon[63913]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3511008014"}]: dispatch 2024-09-13T23:36:13.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:13 smithi022 ceph-mon[80039]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3511008014"}]': finished 2024-09-13T23:36:13.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:13 smithi022 ceph-mon[80039]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T23:36:13.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:13 smithi022 ceph-mon[80039]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:36:13.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:13 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/2806707309' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4289796476"}]: dispatch 2024-09-13T23:36:13.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:13 smithi022 ceph-mon[80039]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4289796476"}]: dispatch 2024-09-13T23:36:13.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:13 smithi022 ceph-mon[82397]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3511008014"}]': finished 2024-09-13T23:36:13.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:13 smithi022 ceph-mon[82397]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T23:36:13.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:13 smithi022 ceph-mon[82397]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:36:13.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:13 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/2806707309' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4289796476"}]: dispatch 2024-09-13T23:36:13.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:13 smithi022 ceph-mon[82397]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4289796476"}]: dispatch 2024-09-13T23:36:13.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:13 smithi076 ceph-mon[63913]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/3511008014"}]': finished 2024-09-13T23:36:13.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:13 smithi076 ceph-mon[63913]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T23:36:13.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:13 smithi076 ceph-mon[63913]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-13T23:36:13.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:13 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/2806707309' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4289796476"}]: dispatch 2024-09-13T23:36:13.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:13 smithi076 ceph-mon[63913]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4289796476"}]: dispatch 2024-09-13T23:36:14.149 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:36:14.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:14 smithi022 ceph-mon[82397]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4289796476"}]': finished 2024-09-13T23:36:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:14 smithi022 ceph-mon[82397]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T23:36:14.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:14 smithi022 ceph-mon[82397]: from='client.64798 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:36:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:14 smithi022 ceph-mon[80039]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4289796476"}]': finished 2024-09-13T23:36:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:14 smithi022 ceph-mon[80039]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T23:36:14.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:14 smithi022 ceph-mon[80039]: from='client.64798 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:36:14.503 INFO:teuthology.orchestra.run.smithi022.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi022.wguwxm on host smithi022 failed." 2024-09-13T23:36:14.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:14 smithi076 ceph-mon[63913]: from='client.? ' entity='client.iscsi.foo.smithi022.wguwxm' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.22:0/4289796476"}]': finished 2024-09-13T23:36:14.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:14 smithi076 ceph-mon[63913]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T23:36:14.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:14 smithi076 ceph-mon[63913]: from='client.64798 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:36:15.234 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:36:15.546 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:15 smithi022 ceph-mon[80039]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:15.546 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:15 smithi022 ceph-mon[80039]: from='client.54870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:36:15.546 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:15 smithi022 ceph-mon[82397]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:15.546 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:15 smithi022 ceph-mon[82397]: from='client.54870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:36:15.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:15 smithi076 ceph-mon[63913]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:15.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:15 smithi076 ceph-mon[63913]: from='client.54870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:36:17.164 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:36:17.164 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (23m) 74s ago 31m 24.8M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:36:17.164 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (10m) 75s ago 31m 82.2M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:36:17.164 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (10m) 74s ago 30m 61.0M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:36:17.164 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (10m) 75s ago 36m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:36:17.164 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (20m) 74s ago 37m 508M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (8m) 74s ago 37m 61.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (9m) 75s ago 36m 51.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (7m) 74s ago 36m 49.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (23m) 74s ago 32m 15.6M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (23m) 75s ago 32m 15.8M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (6m) 74s ago 35m 59.2M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (6m) 74s ago 35m 61.2M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (5m) 74s ago 34m 55.3M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 42db9ab27cb1 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (4m) 74s ago 34m 53.5M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 b891533f55d3 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (3m) 75s ago 34m 57.6M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 ba143ef88b4e 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (3m) 75s ago 33m 55.4M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 57c6e1a1827d 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (2m) 75s ago 33m 49.6M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 c0ab477f2b96 2024-09-13T23:36:17.165 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (2m) 75s ago 32m 54.5M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 4325dc4e4077 2024-09-13T23:36:17.166 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (10m) 75s ago 32m 69.6M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:36:17.166 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (84s) 74s ago 30m 77.8M - 19.1.1-324-gecab7be5 0652ff2b5d78 eb4e10d75d7f 2024-09-13T23:36:17.166 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (79s) 75s ago 30m 75.0M - 19.1.1-324-gecab7be5 0652ff2b5d78 833b94cee8f5 2024-09-13T23:36:17.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:36:16 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=cleanup t=2024-09-13T23:36:16.822954271Z level=info msg="Completed cleanup jobs" duration=2.551843ms 2024-09-13T23:36:17.713 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:17 smithi022 ceph-mon[80039]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:17.713 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:17 smithi022 ceph-mon[80039]: from='client.64810 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:36:17.713 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:17 smithi022 ceph-mon[80039]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:17.713 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:17 smithi022 ceph-mon[82397]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:17.713 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:17 smithi022 ceph-mon[82397]: from='client.64810 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:36:17.714 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:17 smithi022 ceph-mon[82397]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:17.737 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T23:36:17.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:17 smithi076 ceph-mon[63913]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:17.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:17 smithi076 ceph-mon[63913]: from='client.64810 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:36:17.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:17 smithi076 ceph-mon[63913]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:19.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:19 smithi022 ceph-mon[80039]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T23:36:19.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:19 smithi022 ceph-mon[82397]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T23:36:19.765 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:36:19.765 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:36:19.765 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 15 2024-09-13T23:36:19.766 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:36:19.767 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:36:19.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:19 smithi076 ceph-mon[63913]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T23:36:20.434 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-13T23:36:20.677 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:20 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/2153885269' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:36:20.678 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:20 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/2153885269' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:36:20.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:20 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/2153885269' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:36:21.741 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:21 smithi022 ceph-mon[80039]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:21.741 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:36:21.742 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:21 smithi022 ceph-mon[82397]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:21.742 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:36:21.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:21 smithi076 ceph-mon[63913]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:21.787 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:36:22.227 INFO:teuthology.orchestra.run.smithi022.stdout:wait for servicemap items w/ changing names to refresh 2024-09-13T23:36:22.848 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:36:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:36:22] "GET /metrics HTTP/1.1" 200 38425 "" "Prometheus/2.43.0" 2024-09-13T23:36:22.878 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 60' 2024-09-13T23:36:23.788 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:23 smithi076 ceph-mon[63913]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:36:23.838 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:23 smithi022 ceph-mon[80039]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:36:23.838 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:23 smithi022 ceph-mon[82397]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T23:36:26.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:25 smithi022 ceph-mon[80039]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s 2024-09-13T23:36:26.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:25 smithi022 ceph-mon[82397]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s 2024-09-13T23:36:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:25 smithi076 ceph-mon[63913]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s 2024-09-13T23:36:28.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:27 smithi022 ceph-mon[80039]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:28.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:27 smithi022 ceph-mon[80039]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:28.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:27 smithi022 ceph-mon[82397]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:28.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:27 smithi022 ceph-mon[82397]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:27 smithi076 ceph-mon[63913]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:27 smithi076 ceph-mon[63913]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:30.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:29 smithi022 ceph-mon[80039]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:30.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:29 smithi022 ceph-mon[82397]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:29 smithi076 ceph-mon[63913]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:32.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:31 smithi022 ceph-mon[80039]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:32.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:31 smithi022 ceph-mon[82397]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:32.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:31 smithi076 ceph-mon[63913]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:32.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:36:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:36:32] "GET /metrics HTTP/1.1" 200 38425 "" "Prometheus/2.43.0" 2024-09-13T23:36:34.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:33 smithi022 ceph-mon[80039]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:33 smithi022 ceph-mon[82397]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:33 smithi076 ceph-mon[63913]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:36.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:35 smithi022 ceph-mon[80039]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:36.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:35 smithi022 ceph-mon[82397]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:35 smithi076 ceph-mon[63913]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:37.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:36:37.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:36:37.188 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:36:38.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:37 smithi022 ceph-mon[82397]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:38.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:37 smithi022 ceph-mon[82397]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:38.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:37 smithi022 ceph-mon[80039]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:38.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:37 smithi022 ceph-mon[80039]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:37 smithi076 ceph-mon[63913]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:37 smithi076 ceph-mon[63913]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:40.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:39 smithi022 ceph-mon[82397]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:40.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:39 smithi022 ceph-mon[80039]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:40.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:39 smithi076 ceph-mon[63913]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:42.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:41 smithi022 ceph-mon[80039]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:42.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:41 smithi022 ceph-mon[82397]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:41 smithi076 ceph-mon[63913]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:42.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:36:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:36:42] "GET /metrics HTTP/1.1" 200 38421 "" "Prometheus/2.43.0" 2024-09-13T23:36:44.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:43 smithi022 ceph-mon[80039]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:44.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:43 smithi022 ceph-mon[82397]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:43 smithi076 ceph-mon[63913]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:46.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:45 smithi022 ceph-mon[80039]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:46.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:45 smithi022 ceph-mon[82397]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:45 smithi076 ceph-mon[63913]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:48.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:47 smithi022 ceph-mon[82397]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:48.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:47 smithi022 ceph-mon[82397]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:48.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:47 smithi022 ceph-mon[80039]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:48.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:47 smithi022 ceph-mon[80039]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:48.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:47 smithi076 ceph-mon[63913]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:48.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:47 smithi076 ceph-mon[63913]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:50.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:49 smithi022 ceph-mon[82397]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:50.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:49 smithi022 ceph-mon[80039]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:49 smithi076 ceph-mon[63913]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:52.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:51 smithi022 ceph-mon[80039]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:52.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:51 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:36:52.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:51 smithi022 ceph-mon[82397]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:52.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:51 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:36:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:51 smithi076 ceph-mon[63913]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:51 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:36:52.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:36:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:36:52] "GET /metrics HTTP/1.1" 200 38423 "" "Prometheus/2.43.0" 2024-09-13T23:36:54.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:53 smithi022 ceph-mon[80039]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:54.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:53 smithi022 ceph-mon[82397]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:54.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:53 smithi076 ceph-mon[63913]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:56.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:55 smithi022 ceph-mon[80039]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:56.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:55 smithi022 ceph-mon[82397]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:56.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:55 smithi076 ceph-mon[63913]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:36:58.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:57 smithi022 ceph-mon[82397]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:58.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:36:58.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:36:58.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:57 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:36:58.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:57 smithi022 ceph-mon[82397]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:58.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:57 smithi022 ceph-mon[80039]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:58.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:36:58.178 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:36:58.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:57 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:36:58.179 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:57 smithi022 ceph-mon[80039]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:36:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:57 smithi076 ceph-mon[63913]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:36:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T23:36:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T23:36:58.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:57 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' 2024-09-13T23:36:58.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:57 smithi076 ceph-mon[63913]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:00.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:36:59 smithi022 ceph-mon[80039]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:00.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:36:59 smithi022 ceph-mon[82397]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:00.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:36:59 smithi076 ceph-mon[63913]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:02.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:01 smithi022 ceph-mon[80039]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:02.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:01 smithi022 ceph-mon[82397]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:02.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:01 smithi076 ceph-mon[63913]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:02.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:02 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:37:02] "GET /metrics HTTP/1.1" 200 38423 "" "Prometheus/2.43.0" 2024-09-13T23:37:04.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:03 smithi022 ceph-mon[82397]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:04.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:03 smithi022 ceph-mon[80039]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:04.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:03 smithi076 ceph-mon[63913]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:06.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:05 smithi022 ceph-mon[80039]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:06.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:05 smithi022 ceph-mon[82397]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:06.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:05 smithi076 ceph-mon[63913]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:07.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:06 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:37:07.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:06 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:37:07.188 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:06 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:37:08.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:07 smithi022 ceph-mon[80039]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:08.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:07 smithi022 ceph-mon[80039]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:07 smithi022 ceph-mon[82397]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:08.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:07 smithi022 ceph-mon[82397]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:07 smithi076 ceph-mon[63913]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:08.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:07 smithi076 ceph-mon[63913]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:10.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:09 smithi022 ceph-mon[80039]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:10.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:09 smithi022 ceph-mon[82397]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:10.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:09 smithi076 ceph-mon[63913]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:12.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:11 smithi022 ceph-mon[80039]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:12.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:11 smithi022 ceph-mon[82397]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:12.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:11 smithi076 ceph-mon[63913]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:12.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:12 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:37:12] "GET /metrics HTTP/1.1" 200 38421 "" "Prometheus/2.43.0" 2024-09-13T23:37:14.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:13 smithi022 ceph-mon[80039]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:14.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:13 smithi022 ceph-mon[82397]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:14.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:13 smithi076 ceph-mon[63913]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:16.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:15 smithi022 ceph-mon[80039]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:16.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:15 smithi022 ceph-mon[82397]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:16.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:15 smithi076 ceph-mon[63913]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:18.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:17 smithi022 ceph-mon[80039]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:18.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:17 smithi022 ceph-mon[80039]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:18.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:17 smithi022 ceph-mon[82397]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:18.178 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:17 smithi022 ceph-mon[82397]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:17 smithi076 ceph-mon[63913]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:18.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:17 smithi076 ceph-mon[63913]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:20.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:19 smithi022 ceph-mon[80039]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:20.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:19 smithi022 ceph-mon[82397]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:20.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:19 smithi076 ceph-mon[63913]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:22.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:21 smithi022 ceph-mon[80039]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:37:22.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:21 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:37:22.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:21 smithi022 ceph-mon[82397]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:37:22.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:21 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:37:22.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:21 smithi076 ceph-mon[63913]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:37:22.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:21 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:37:22.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:22 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:37:22] "GET /metrics HTTP/1.1" 200 38425 "" "Prometheus/2.43.0" 2024-09-13T23:37:24.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:23 smithi022 ceph-mon[80039]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:37:24.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:23 smithi022 ceph-mon[82397]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:37:24.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:23 smithi076 ceph-mon[63913]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-13T23:37:25.363 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T23:37:25.961 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:25 smithi022 ceph-mon[80039]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:25.961 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:25 smithi022 ceph-mon[82397]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:26.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:25 smithi076 ceph-mon[63913]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:27.451 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T23:37:27.451 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (24m) 2m ago 32m 24.8M - 0.25.0 c8568f914cd2 dbd41932f474 2024-09-13T23:37:27.451 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi076 *:3000 running (11m) 2m ago 32m 82.2M - 9.4.12 2bacad6d85d8 79498011f0b3 2024-09-13T23:37:27.451 INFO:teuthology.orchestra.run.smithi022.stdout:iscsi.foo.smithi022.wguwxm smithi022 running (11m) 2m ago 31m 61.0M - 3.5 e1d6a67b021e a6a52a250049 2024-09-13T23:37:27.451 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi076 *:8443,9283,8765 running (11m) 2m ago 37m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 8f0261a79cc0 2024-09-13T23:37:27.451 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (21m) 2m ago 38m 508M - 19.1.1-324-gecab7be5 0652ff2b5d78 9d08241cbfbf 2024-09-13T23:37:27.451 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (9m) 2m ago 39m 61.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 560b94ba5a9b 2024-09-13T23:37:27.451 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi076 running (10m) 2m ago 37m 51.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 65c65bdd515a 2024-09-13T23:37:27.452 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (9m) 2m ago 37m 49.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 daaa9197af04 2024-09-13T23:37:27.452 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (24m) 2m ago 33m 15.6M - 1.5.0 0da6a335fe13 892ef92b2fce 2024-09-13T23:37:27.452 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi076 *:9100 running (24m) 2m ago 33m 15.8M - 1.5.0 0da6a335fe13 1141a03b8cd7 2024-09-13T23:37:27.452 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (7m) 2m ago 36m 59.2M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 ad68a4505113 2024-09-13T23:37:27.452 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (7m) 2m ago 36m 61.2M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 866b71c23f0a 2024-09-13T23:37:27.452 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (6m) 2m ago 36m 55.3M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 42db9ab27cb1 2024-09-13T23:37:27.452 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (5m) 2m ago 35m 53.5M 2993M 19.1.1-324-gecab7be5 0652ff2b5d78 b891533f55d3 2024-09-13T23:37:27.452 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi076 running (4m) 2m ago 35m 57.6M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 ba143ef88b4e 2024-09-13T23:37:27.452 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi076 running (4m) 2m ago 34m 55.4M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 57c6e1a1827d 2024-09-13T23:37:27.452 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi076 running (3m) 2m ago 34m 49.6M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 c0ab477f2b96 2024-09-13T23:37:27.452 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi076 running (3m) 2m ago 34m 54.5M 3248M 19.1.1-324-gecab7be5 0652ff2b5d78 4325dc4e4077 2024-09-13T23:37:27.453 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi076 *:9095 running (11m) 2m ago 33m 69.6M - 2.43.0 a07b618ecd1d 6f8a2fca1117 2024-09-13T23:37:27.453 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi022.snmrkm smithi022 *:8000 running (2m) 2m ago 31m 77.8M - 19.1.1-324-gecab7be5 0652ff2b5d78 eb4e10d75d7f 2024-09-13T23:37:27.453 INFO:teuthology.orchestra.run.smithi022.stdout:rgw.foo.smithi076.rnamrn smithi076 *:8000 running (2m) 2m ago 31m 75.0M - 19.1.1-324-gecab7be5 0652ff2b5d78 833b94cee8f5 2024-09-13T23:37:28.030 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:27 smithi022 ceph-mon[80039]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:28.030 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:27 smithi022 ceph-mon[80039]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:28.030 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:27 smithi022 ceph-mon[82397]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:28.030 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:27 smithi022 ceph-mon[82397]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:28.054 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T23:37:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:27 smithi076 ceph-mon[63913]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:28.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:27 smithi076 ceph-mon[63913]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:29.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:28 smithi076 ceph-mon[63913]: from='client.64822 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:29.362 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:28 smithi022 ceph-mon[80039]: from='client.64822 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:29.363 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:28 smithi022 ceph-mon[82397]: from='client.64822 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:30.157 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:37:30.158 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-13T23:37:30.158 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T23:37:30.158 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:37:30.158 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-13T23:37:30.158 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:37:30.158 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:37:30.158 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-13T23:37:30.158 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T23:37:30.158 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:37:30.158 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw": { 2024-09-13T23:37:30.159 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T23:37:30.159 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-13T23:37:30.159 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-13T23:37:30.159 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 15 2024-09-13T23:37:30.159 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-13T23:37:30.159 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:37:30.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:29 smithi076 ceph-mon[63913]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:30.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:29 smithi022 ceph-mon[82397]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:30.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:29 smithi022 ceph-mon[80039]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:30.752 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T23:37:31.053 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:30 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/1158196539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:37:31.054 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:30 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/1158196539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:37:31.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:30 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/1158196539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:37:31.976 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:31 smithi022 ceph-mon[80039]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:31.976 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:31 smithi022 ceph-mon[82397]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:32.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:31 smithi076 ceph-mon[63913]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:32.677 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:32 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:37:32] "GET /metrics HTTP/1.1" 200 38425 "" "Prometheus/2.43.0" 2024-09-13T23:37:32.707 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:37:32.707 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T23:37:32.707 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-13T23:37:32.707 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T23:37:32.708 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-13T23:37:32.708 INFO:teuthology.orchestra.run.smithi022.stdout: "osd", 2024-09-13T23:37:32.708 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr", 2024-09-13T23:37:32.708 INFO:teuthology.orchestra.run.smithi022.stdout: "rgw", 2024-09-13T23:37:32.708 INFO:teuthology.orchestra.run.smithi022.stdout: "mon" 2024-09-13T23:37:32.708 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-13T23:37:32.708 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "15/21 daemons upgraded", 2024-09-13T23:37:32.708 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi022.wguwxm on host smithi022 failed.", 2024-09-13T23:37:32.708 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": true 2024-09-13T23:37:32.708 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:37:33.310 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T23:37:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:33 smithi022 ceph-mon[82397]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:34.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:33 smithi022 ceph-mon[82397]: from='client.54900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:34.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:33 smithi022 ceph-mon[80039]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:34.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:33 smithi022 ceph-mon[80039]: from='client.54900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:33 smithi076 ceph-mon[63913]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:34.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:33 smithi076 ceph-mon[63913]: from='client.54900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:35.326 INFO:teuthology.orchestra.run.smithi022.stdout:HEALTH_WARN Upgrading daemon iscsi.foo.smithi022.wguwxm on host smithi022 failed. 2024-09-13T23:37:35.327 INFO:teuthology.orchestra.run.smithi022.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi022.wguwxm on host smithi022 failed. 2024-09-13T23:37:35.327 INFO:teuthology.orchestra.run.smithi022.stdout: Upgrade daemon: iscsi.foo.smithi022.wguwxm: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi022.wguwxm ... 2024-09-13T23:37:35.327 INFO:teuthology.orchestra.run.smithi022.stdout:Creating ceph-iscsi config... 2024-09-13T23:37:35.327 INFO:teuthology.orchestra.run.smithi022.stdout:Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/iscsi-gateway.cfg 2024-09-13T23:37:35.327 INFO:teuthology.orchestra.run.smithi022.stdout:Write file: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/iscsi.foo.smithi022.wguwxm/tcmu-runner-entrypoint.sh 2024-09-13T23:37:35.327 INFO:teuthology.orchestra.run.smithi022.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2d9adcac8c\x2d7223\x2d11ef\x2dbceb\x2dc7b262605968.slice/ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service 2024-09-13T23:37:35.327 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 1 from systemctl start ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm 2024-09-13T23:37:35.328 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stderr Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:37:35.328 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stderr See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:37:35.328 INFO:teuthology.orchestra.run.smithi022.stdout:Traceback (most recent call last): 2024-09-13T23:37:35.328 INFO:teuthology.orchestra.run.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T23:37:35.328 INFO:teuthology.orchestra.run.smithi022.stdout: return _run_code(code, main_globals, None, 2024-09-13T23:37:35.328 INFO:teuthology.orchestra.run.smithi022.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T23:37:35.329 INFO:teuthology.orchestra.run.smithi022.stdout: exec(code, run_globals) 2024-09-13T23:37:35.329 INFO:teuthology.orchestra.run.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T23:37:35.329 INFO:teuthology.orchestra.run.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T23:37:35.329 INFO:teuthology.orchestra.run.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T23:37:35.329 INFO:teuthology.orchestra.run.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T23:37:35.329 INFO:teuthology.orchestra.run.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T23:37:35.329 INFO:teuthology.orchestra.run.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1088, in deploy_daemon 2024-09-13T23:37:35.329 INFO:teuthology.orchestra.run.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1224, in deploy_daemon_units 2024-09-13T23:37:35.330 INFO:teuthology.orchestra.run.smithi022.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T23:37:35.330 INFO:teuthology.orchestra.run.smithi022.stdout:RuntimeError: Failed command: systemctl start ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm: Job for ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service failed because the control process exited with error code. 2024-09-13T23:37:35.330 INFO:teuthology.orchestra.run.smithi022.stdout:See "systemctl status ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" and "journalctl -xeu ceph-9adcac8c-7223-11ef-bceb-c7b262605968@iscsi.foo.smithi022.wguwxm.service" for details. 2024-09-13T23:37:35.967 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-13T23:37:36.277 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:35 smithi022 ceph-mon[80039]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:36.277 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:35 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/398714125' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:37:36.278 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:35 smithi022 ceph-mon[82397]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:36.278 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:35 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/398714125' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:37:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:35 smithi076 ceph-mon[63913]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:36.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:35 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/398714125' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T23:37:37.204 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:36 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:37:37.204 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:36 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:37:37.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:36 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:37:38.094 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-13T23:37:38.174 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:37 smithi022 ceph-mon[80039]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:38.174 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:37 smithi022 ceph-mon[80039]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:38.174 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:37 smithi022 ceph-mon[82397]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:38.175 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:37 smithi022 ceph-mon[82397]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:37 smithi076 ceph-mon[63913]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:38.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:37 smithi076 ceph-mon[63913]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:38.747 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-13T23:37:39.057 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:38 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/2659609409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:37:39.057 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:38 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/2659609409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:37:39.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:38 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/2659609409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:37:40.200 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:39 smithi022 ceph-mon[80039]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:40.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:39 smithi022 ceph-mon[82397]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:40.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:39 smithi076 ceph-mon[63913]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:40.784 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)" 2024-09-13T23:37:41.075 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:40 smithi022 ceph-mon[80039]: from='client.? 172.21.15.22:0/4118639493' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:37:41.075 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:40 smithi022 ceph-mon[82397]: from='client.? 172.21.15.22:0/4118639493' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:37:41.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:40 smithi076 ceph-mon[63913]: from='client.? 172.21.15.22:0/4118639493' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T23:37:41.405 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-09-13T23:37:42.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:41 smithi022 ceph-mon[80039]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:42.004 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:41 smithi022 ceph-mon[82397]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:42.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:41 smithi076 ceph-mon[63913]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:42.746 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:42 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:37:42] "GET /metrics HTTP/1.1" 200 38429 "" "Prometheus/2.43.0" 2024-09-13T23:37:43.445 INFO:teuthology.orchestra.run.smithi022.stdout:osd 8 2m ago - 2024-09-13T23:37:44.031 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:43 smithi022 ceph-mon[82397]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:44.032 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:43 smithi022 ceph-mon[80039]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:44.058 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T23:37:44.068 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi022.front.sepia.ceph.com 2024-09-13T23:37:44.068 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-09-13T23:37:44.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:43 smithi076 ceph-mon[63913]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:45.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:44 smithi076 ceph-mon[63913]: from='client.54924 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:45.330 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:44 smithi022 ceph-mon[80039]: from='client.54924 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:45.330 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:44 smithi022 ceph-mon[82397]: from='client.54924 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:46.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:45 smithi076 ceph-mon[63913]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:46.400 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-13T23:37:46.401 INFO:teuthology.orchestra.run.smithi022.stdout: "image": "quay.io/ceph/ceph", 2024-09-13T23:37:46.401 INFO:teuthology.orchestra.run.smithi022.stdout: "registry": "quay.io", 2024-09-13T23:37:46.401 INFO:teuthology.orchestra.run.smithi022.stdout: "bare_image": "ceph/ceph", 2024-09-13T23:37:46.401 INFO:teuthology.orchestra.run.smithi022.stdout: "versions": [ 2024-09-13T23:37:46.401 INFO:teuthology.orchestra.run.smithi022.stdout: "19.1.1", 2024-09-13T23:37:46.401 INFO:teuthology.orchestra.run.smithi022.stdout: "19.1.0" 2024-09-13T23:37:46.401 INFO:teuthology.orchestra.run.smithi022.stdout: ] 2024-09-13T23:37:46.401 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-13T23:37:46.402 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:45 smithi022 ceph-mon[82397]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:46.402 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:45 smithi022 ceph-mon[80039]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:47.063 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:46 smithi022 ceph-mon[80039]: from='client.54930 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:47.063 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:46 smithi022 ceph-mon[82397]: from='client.54930 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:47.092 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-09-13T23:37:47.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:46 smithi076 ceph-mon[63913]: from='client.54930 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:48.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:47 smithi076 ceph-mon[63913]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:48.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:47 smithi076 ceph-mon[63913]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:48.378 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:47 smithi022 ceph-mon[82397]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:48.378 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:47 smithi022 ceph-mon[82397]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:48.378 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:47 smithi022 ceph-mon[80039]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:48.378 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:47 smithi022 ceph-mon[80039]: from='client.54816 -' entity='client.iscsi.foo.smithi022.wguwxm' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T23:37:49.458 INFO:teuthology.orchestra.run.smithi022.stdout: "16.2.0", 2024-09-13T23:37:50.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:49 smithi022 ceph-mon[80039]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:50.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:49 smithi022 ceph-mon[80039]: from='client.54936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:50.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:49 smithi022 ceph-mon[82397]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:50.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:49 smithi022 ceph-mon[82397]: from='client.54936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:50.085 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-09-13T23:37:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:49 smithi076 ceph-mon[63913]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T23:37:50.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:49 smithi076 ceph-mon[63913]: from='client.54936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:52.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:51 smithi076 ceph-mon[63913]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:52.288 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:51 smithi076 ceph-mon[63913]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:37:52.356 INFO:teuthology.orchestra.run.smithi022.stdout: "v16.2.2", 2024-09-13T23:37:52.357 INFO:teuthology.orchestra.run.smithi022.stdout: "v16.2.2-20210505", 2024-09-13T23:37:52.357 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:51 smithi022 ceph-mon[80039]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:52.357 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:51 smithi022 ceph-mon[80039]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:37:52.357 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:51 smithi022 ceph-mon[82397]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T23:37:52.357 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:51 smithi022 ceph-mon[82397]: from='mgr.54108 172.21.15.22:0/2001338033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T23:37:52.649 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:52 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: ::ffff:172.21.15.76 - - [13/Sep/2024:23:37:52] "GET /metrics HTTP/1.1" 200 38424 "" "Prometheus/2.43.0" 2024-09-13T23:37:53.018 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T23:37:53.027 INFO:tasks.cephadm:Teardown begin 2024-09-13T23:37:53.028 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T23:37:53.058 DEBUG:teuthology.orchestra.run.smithi076:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T23:37:53.090 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T23:37:53.090 DEBUG:teuthology.orchestra.run.smithi022:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T23:37:53.116 DEBUG:teuthology.orchestra.run.smithi076:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T23:37:53.147 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T23:37:53.147 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-13T23:37:53.147 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.a 2024-09-13T23:37:53.287 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:52 smithi076 ceph-mon[63913]: from='client.64876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:53.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:52 smithi022 ceph-mon[82397]: from='client.64876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:53.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:52 smithi022 ceph-mon[80039]: from='client.64876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T23:37:53.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:53 smithi022 systemd[1]: Stopping Ceph mon.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:37:53.724 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a[80035]: 2024-09-13T23:37:53.456+0000 7f90013ad640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:37:53.724 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a[80035]: 2024-09-13T23:37:53.456+0000 7f90013ad640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-13T23:37:53.724 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:37:53] ENGINE Bus STOPPING 2024-09-13T23:37:54.003 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:53 smithi022 podman[124204]: 2024-09-13 23:37:53.722288061 +0000 UTC m=+0.373015543 container died 560b94ba5a9b31b0e9230cce2a7894891f3be7bdeeee8f5da39490d623f2dfc2 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:37:54.004 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:37:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T23:37:54.004 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:37:53] ENGINE Bus STOPPED 2024-09-13T23:37:54.004 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:53 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:37:53] ENGINE Bus STARTING 2024-09-13T23:37:54.345 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:54 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:37:54] ENGINE Serving on http://:::9283 2024-09-13T23:37:54.345 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:54 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:37:54] ENGINE Bus STARTED 2024-09-13T23:37:54.345 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:54 smithi022 podman[124204]: 2024-09-13 23:37:54.101753064 +0000 UTC m=+0.752480544 container cleanup 560b94ba5a9b31b0e9230cce2a7894891f3be7bdeeee8f5da39490d623f2dfc2 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T23:37:54.346 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 13 23:37:54 smithi022 bash[124204]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-a 2024-09-13T23:37:54.490 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.a.service' 2024-09-13T23:37:54.539 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:37:54.539 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-13T23:37:54.539 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-13T23:37:54.539 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.c 2024-09-13T23:37:54.677 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:54 smithi022 systemd[1]: Stopping Ceph mon.c for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:37:55.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:54 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c[82393]: 2024-09-13T23:37:54.856+0000 7fe513e02640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:37:55.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:54 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c[82393]: 2024-09-13T23:37:54.856+0000 7fe513e02640 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-09-13T23:37:55.177 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:55 smithi022 podman[124336]: 2024-09-13 23:37:55.030078462 +0000 UTC m=+0.299389758 container died daaa9197af0446babb1008e27a539b44470d26f6fe53e571db500a6533ceee69 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:37:55.177 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:54 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:37:54] ENGINE Bus STOPPING 2024-09-13T23:37:55.177 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:37:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T23:37:55.177 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:37:55] ENGINE Bus STOPPED 2024-09-13T23:37:55.177 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:37:55] ENGINE Bus STARTING 2024-09-13T23:37:55.177 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:37:55] ENGINE Serving on http://:::9283 2024-09-13T23:37:55.177 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:55 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: [13/Sep/2024:23:37:55] ENGINE Bus STARTED 2024-09-13T23:37:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:55 smithi022 podman[124336]: 2024-09-13 23:37:55.389530249 +0000 UTC m=+0.658841544 container cleanup daaa9197af0446babb1008e27a539b44470d26f6fe53e571db500a6533ceee69 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-13T23:37:55.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 13 23:37:55 smithi022 bash[124336]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-c 2024-09-13T23:37:55.831 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.c.service' 2024-09-13T23:37:55.866 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:37:55.867 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-13T23:37:55.867 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-13T23:37:55.867 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.b 2024-09-13T23:37:56.280 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:55 smithi076 systemd[1]: Stopping Ceph mon.b for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:37:56.537 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b[63909]: 2024-09-13T23:37:56.279+0000 7f8a90ba8640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:37:56.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b[63909]: 2024-09-13T23:37:56.279+0000 7f8a90ba8640 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-09-13T23:37:56.538 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:56 smithi076 podman[97058]: 2024-09-13 23:37:56.381326604 +0000 UTC m=+0.311594001 container died 65c65bdd515a3c9f75144c876a0851f3f18024755add0f0002d80a37483809ca (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T23:37:57.037 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:56 smithi076 podman[97058]: 2024-09-13 23:37:56.83753647 +0000 UTC m=+0.767803821 container cleanup 65c65bdd515a3c9f75144c876a0851f3f18024755add0f0002d80a37483809ca (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T23:37:57.038 INFO:journalctl@ceph.mon.b.smithi076.stdout:Sep 13 23:37:56 smithi076 bash[97058]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mon-b 2024-09-13T23:37:57.314 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mon.b.service' 2024-09-13T23:37:57.362 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:37:57.362 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-13T23:37:57.362 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-13T23:37:57.362 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.y 2024-09-13T23:37:57.676 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:57 smithi022 systemd[1]: Stopping Ceph mgr.y for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:37:58.012 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:37:57.702+0000 7fd83342c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:37:58.013 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:57 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y[61723]: 2024-09-13T23:37:57.702+0000 7fd83342c640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-13T23:37:58.013 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:57 smithi022 podman[124469]: 2024-09-13 23:37:57.743179462 +0000 UTC m=+0.179755506 container died 9d08241cbfbfde793dceef945accd6dc6f7611f88423a9670797591c9c70513f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T23:37:58.362 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:58 smithi022 podman[124469]: 2024-09-13 23:37:58.143928683 +0000 UTC m=+0.580504727 container cleanup 9d08241cbfbfde793dceef945accd6dc6f7611f88423a9670797591c9c70513f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-13T23:37:58.362 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 13 23:37:58 smithi022 bash[124469]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-y 2024-09-13T23:37:58.503 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-13T23:37:58.539 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:37:58.539 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-13T23:37:58.539 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-13T23:37:58.539 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.x 2024-09-13T23:37:58.866 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:37:58 smithi076 systemd[1]: Stopping Ceph mgr.x for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:37:59.135 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:37:58 smithi076 podman[97180]: 2024-09-13 23:37:58.865009197 +0000 UTC m=+0.137911995 container died 8f0261a79cc0e18d3efedfd9cb0e0d326499a1c2ed3686ccc2fef7d44f19cf6c (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:37:59.494 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:37:59 smithi076 podman[97180]: 2024-09-13 23:37:59.300599713 +0000 UTC m=+0.573502519 container cleanup 8f0261a79cc0e18d3efedfd9cb0e0d326499a1c2ed3686ccc2fef7d44f19cf6c (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-13T23:37:59.494 INFO:journalctl@ceph.mgr.x.smithi076.stdout:Sep 13 23:37:59 smithi076 bash[97180]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-mgr-x 2024-09-13T23:37:59.716 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-13T23:37:59.750 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:37:59.751 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-13T23:37:59.751 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T23:37:59.751 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.0 2024-09-13T23:38:00.040 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:37:59 smithi022 systemd[1]: Stopping Ceph osd.0 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:38:00.427 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:00 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0[89719]: 2024-09-13T23:38:00.038+0000 7f037daef640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:38:00.427 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:00 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0[89719]: 2024-09-13T23:38:00.038+0000 7f037daef640 -1 osd.0 159 *** Got signal Terminated *** 2024-09-13T23:38:00.427 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:00 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0[89719]: 2024-09-13T23:38:00.038+0000 7f037daef640 -1 osd.0 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:38:05.782 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:05 smithi022 podman[124590]: 2024-09-13 23:38:05.515441613 +0000 UTC m=+5.565273458 container died ad68a450511362fd57ebf9de49531cfc3d3c592d023331a0074ec2a2751d0445 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:38:06.141 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:05 smithi022 podman[124590]: 2024-09-13 23:38:05.889424478 +0000 UTC m=+5.939256320 container cleanup ad68a450511362fd57ebf9de49531cfc3d3c592d023331a0074ec2a2751d0445 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-13T23:38:06.141 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:05 smithi022 bash[124590]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0 2024-09-13T23:38:06.426 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:06 smithi022 podman[124606]: 2024-09-13 23:38:06.140571673 +0000 UTC m=+0.620314596 container remove ad68a450511362fd57ebf9de49531cfc3d3c592d023331a0074ec2a2751d0445 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-13T23:38:06.761 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:06 smithi022 podman[124697]: 2024-09-13 23:38:06.364792824 +0000 UTC m=+0.016581205 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:38:06.761 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:06 smithi022 podman[124697]: 2024-09-13 23:38:06.483033619 +0000 UTC m=+0.134822032 container create 6d6a458689da21b5c47a061e30b3df4f3a7014f4890598f64b2f5573184ab9bb (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:38:06.761 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:06 smithi022 podman[124697]: 2024-09-13 23:38:06.677213398 +0000 UTC m=+0.329001772 container init 6d6a458689da21b5c47a061e30b3df4f3a7014f4890598f64b2f5573184ab9bb (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-13T23:38:06.761 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:06 smithi022 podman[124697]: 2024-09-13 23:38:06.680591758 +0000 UTC m=+0.332380127 container start 6d6a458689da21b5c47a061e30b3df4f3a7014f4890598f64b2f5573184ab9bb (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T23:38:07.156 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:06 smithi022 podman[124697]: 2024-09-13 23:38:06.760283225 +0000 UTC m=+0.412071597 container attach 6d6a458689da21b5c47a061e30b3df4f3a7014f4890598f64b2f5573184ab9bb (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-ecab7be, io.buildah.version=1.37.2) 2024-09-13T23:38:07.156 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 13 23:38:06 smithi022 podman[124697]: 2024-09-13 23:38:06.874496025 +0000 UTC m=+0.526284399 container died 6d6a458689da21b5c47a061e30b3df4f3a7014f4890598f64b2f5573184ab9bb (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T23:38:07.435 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T23:38:07.476 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:38:07.477 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T23:38:07.477 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T23:38:07.477 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.1 2024-09-13T23:38:07.673 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:07 smithi022 systemd[1]: Stopping Ceph osd.1 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:38:07.927 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1[94923]: 2024-09-13T23:38:07.825+0000 7f3e175ea640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:38:07.927 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1[94923]: 2024-09-13T23:38:07.825+0000 7f3e175ea640 -1 osd.1 159 *** Got signal Terminated *** 2024-09-13T23:38:07.927 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1[94923]: 2024-09-13T23:38:07.825+0000 7f3e175ea640 -1 osd.1 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:38:13.427 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:13 smithi022 podman[124799]: 2024-09-13 23:38:13.173715809 +0000 UTC m=+5.470665178 container died 866b71c23f0a390ff3d4a1ca0d2765f513d58c86dadccd8bc04480488906bc17 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T23:38:13.735 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:13 smithi022 podman[124799]: 2024-09-13 23:38:13.517161723 +0000 UTC m=+5.814111082 container cleanup 866b71c23f0a390ff3d4a1ca0d2765f513d58c86dadccd8bc04480488906bc17 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2) 2024-09-13T23:38:13.735 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:13 smithi022 bash[124799]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1 2024-09-13T23:38:14.144 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:13 smithi022 podman[124827]: 2024-09-13 23:38:13.734671724 +0000 UTC m=+0.557928578 container remove 866b71c23f0a390ff3d4a1ca0d2765f513d58c86dadccd8bc04480488906bc17 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-13T23:38:14.405 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:14 smithi022 podman[124918]: 2024-09-13 23:38:14.04657724 +0000 UTC m=+0.016642165 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:38:14.406 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:14 smithi022 podman[124918]: 2024-09-13 23:38:14.160409597 +0000 UTC m=+0.130474515 container create 885e0afb1b69e9525e684257df89a73e115c3c217784541cc624621a29f3b1d5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-13T23:38:14.406 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:14 smithi022 podman[124918]: 2024-09-13 23:38:14.404936577 +0000 UTC m=+0.375001489 container init 885e0afb1b69e9525e684257df89a73e115c3c217784541cc624621a29f3b1d5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T23:38:14.677 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:14 smithi022 podman[124918]: 2024-09-13 23:38:14.408593234 +0000 UTC m=+0.378658146 container start 885e0afb1b69e9525e684257df89a73e115c3c217784541cc624621a29f3b1d5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T23:38:14.677 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:14 smithi022 podman[124918]: 2024-09-13 23:38:14.513272834 +0000 UTC m=+0.483337759 container attach 885e0afb1b69e9525e684257df89a73e115c3c217784541cc624621a29f3b1d5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-13T23:38:14.677 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:14 smithi022 podman[124918]: 2024-09-13 23:38:14.605634235 +0000 UTC m=+0.575699158 container died 885e0afb1b69e9525e684257df89a73e115c3c217784541cc624621a29f3b1d5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=squid-ecab7be, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T23:38:15.110 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 13 23:38:15 smithi022 podman[124941]: 2024-09-13 23:38:15.091545041 +0000 UTC m=+0.475412774 container remove 885e0afb1b69e9525e684257df89a73e115c3c217784541cc624621a29f3b1d5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-13T23:38:15.116 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T23:38:15.160 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:38:15.160 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T23:38:15.161 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T23:38:15.161 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.2 2024-09-13T23:38:15.427 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:15 smithi022 systemd[1]: Stopping Ceph osd.2 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:38:15.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2[101467]: 2024-09-13T23:38:15.493+0000 7f481f0cf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:38:15.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2[101467]: 2024-09-13T23:38:15.493+0000 7f481f0cf640 -1 osd.2 159 *** Got signal Terminated *** 2024-09-13T23:38:15.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:15 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2[101467]: 2024-09-13T23:38:15.493+0000 7f481f0cf640 -1 osd.2 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:38:21.222 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:20 smithi022 podman[125020]: 2024-09-13 23:38:20.964105646 +0000 UTC m=+5.577925600 container died 42db9ab27cb17f01e810b63b9045f918477be45876dc3bcc98ef99326601c33b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T23:38:21.596 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:21 smithi022 podman[125020]: 2024-09-13 23:38:21.32762136 +0000 UTC m=+5.941441308 container cleanup 42db9ab27cb17f01e810b63b9045f918477be45876dc3bcc98ef99326601c33b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-13T23:38:21.596 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:21 smithi022 bash[125020]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2 2024-09-13T23:38:21.927 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:21 smithi022 podman[125035]: 2024-09-13 23:38:21.595464267 +0000 UTC m=+0.632519530 container remove 42db9ab27cb17f01e810b63b9045f918477be45876dc3bcc98ef99326601c33b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:38:22.249 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:21 smithi022 podman[125125]: 2024-09-13 23:38:21.962819094 +0000 UTC m=+0.113283571 container create 1067697fa22bec964df94d5793afb645401446c53bd3e01672d558d0d240bd2c (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T23:38:22.249 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:21 smithi022 podman[125125]: 2024-09-13 23:38:21.865690266 +0000 UTC m=+0.016154756 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:38:22.249 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:22 smithi022 podman[125125]: 2024-09-13 23:38:22.173522022 +0000 UTC m=+0.323986501 container init 1067697fa22bec964df94d5793afb645401446c53bd3e01672d558d0d240bd2c (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-13T23:38:22.250 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:22 smithi022 podman[125125]: 2024-09-13 23:38:22.176924455 +0000 UTC m=+0.327388936 container start 1067697fa22bec964df94d5793afb645401446c53bd3e01672d558d0d240bd2c (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T23:38:22.636 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:22 smithi022 podman[125125]: 2024-09-13 23:38:22.248443981 +0000 UTC m=+0.398908478 container attach 1067697fa22bec964df94d5793afb645401446c53bd3e01672d558d0d240bd2c (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-13T23:38:22.637 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:22 smithi022 podman[125125]: 2024-09-13 23:38:22.369302101 +0000 UTC m=+0.519766585 container died 1067697fa22bec964df94d5793afb645401446c53bd3e01672d558d0d240bd2c (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-13T23:38:22.889 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 13 23:38:22 smithi022 podman[125148]: 2024-09-13 23:38:22.876762607 +0000 UTC m=+0.501666501 container remove 1067697fa22bec964df94d5793afb645401446c53bd3e01672d558d0d240bd2c (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T23:38:22.894 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T23:38:22.937 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:38:22.984 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T23:38:22.984 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T23:38:22.985 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.3 2024-09-13T23:38:23.176 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:23 smithi022 systemd[1]: Stopping Ceph osd.3 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:38:23.677 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:23 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[108033]: 2024-09-13T23:38:23.279+0000 7f507f125640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:38:23.677 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:23 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[108033]: 2024-09-13T23:38:23.279+0000 7f507f125640 -1 osd.3 159 *** Got signal Terminated *** 2024-09-13T23:38:23.677 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:23 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[108033]: 2024-09-13T23:38:23.279+0000 7f507f125640 -1 osd.3 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:38:26.541 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:26 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:26.252+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:27.037 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:26 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:26.572+0000 7f212b465640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:00.324693+0000 front 2024-09-13T23:38:00.324746+0000 (oldest deadline 2024-09-13T23:38:26.224644+0000) 2024-09-13T23:38:27.537 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:27.228+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:27.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:27.257+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:28.037 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:27 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:27.605+0000 7f212b465640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:00.324693+0000 front 2024-09-13T23:38:00.324746+0000 (oldest deadline 2024-09-13T23:38:26.224644+0000) 2024-09-13T23:38:28.426 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:28 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3[108033]: 2024-09-13T23:38:28.057+0000 7f507b73b640 -1 osd.3 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.179960+0000 front 2024-09-13T23:38:03.179805+0000 (oldest deadline 2024-09-13T23:38:27.279969+0000) 2024-09-13T23:38:28.537 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:28 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:28.209+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:28.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:28 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:28.297+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:29.037 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:28 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:28.646+0000 7f212b465640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:00.324693+0000 front 2024-09-13T23:38:00.324746+0000 (oldest deadline 2024-09-13T23:38:26.224644+0000) 2024-09-13T23:38:29.080 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:28 smithi022 podman[125227]: 2024-09-13 23:38:28.810381166 +0000 UTC m=+5.638327779 container died b891533f55d351154599368044551ccaec8d701db7096344052e804c9d6a83aa (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T23:38:29.427 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:29 smithi022 podman[125227]: 2024-09-13 23:38:29.179110265 +0000 UTC m=+6.007056875 container cleanup b891533f55d351154599368044551ccaec8d701db7096344052e804c9d6a83aa (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:38:29.427 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:29 smithi022 bash[125227]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3 2024-09-13T23:38:29.537 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:29 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:29.169+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:29.538 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:29 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:29.337+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:29.803 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:29 smithi022 podman[125243]: 2024-09-13 23:38:29.430404612 +0000 UTC m=+0.617420734 container remove b891533f55d351154599368044551ccaec8d701db7096344052e804c9d6a83aa (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:38:29.902 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:29 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:29.602+0000 7f212b465640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:00.324693+0000 front 2024-09-13T23:38:00.324746+0000 (oldest deadline 2024-09-13T23:38:26.224644+0000) 2024-09-13T23:38:30.075 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:29 smithi022 podman[125332]: 2024-09-13 23:38:29.705596275 +0000 UTC m=+0.016379461 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:38:30.076 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:29 smithi022 podman[125332]: 2024-09-13 23:38:29.80605015 +0000 UTC m=+0.116833365 container create 51a6ad3206628bd6330711d0c20e8a47f8dec790825cb379a16d7c251b1dec5e (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-13T23:38:30.076 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:30 smithi022 podman[125332]: 2024-09-13 23:38:29.999948285 +0000 UTC m=+0.310731459 container init 51a6ad3206628bd6330711d0c20e8a47f8dec790825cb379a16d7c251b1dec5e (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T23:38:30.076 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:30 smithi022 podman[125332]: 2024-09-13 23:38:30.003331139 +0000 UTC m=+0.314114315 container start 51a6ad3206628bd6330711d0c20e8a47f8dec790825cb379a16d7c251b1dec5e (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, io.buildah.version=1.37.2) 2024-09-13T23:38:30.076 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:30 smithi022 podman[125332]: 2024-09-13 23:38:30.074846816 +0000 UTC m=+0.385629997 container attach 51a6ad3206628bd6330711d0c20e8a47f8dec790825cb379a16d7c251b1dec5e (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:38:30.210 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:29 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:29.901+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:30.427 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 13 23:38:30 smithi022 podman[125332]: 2024-09-13 23:38:30.205073475 +0000 UTC m=+0.515856659 container died 51a6ad3206628bd6330711d0c20e8a47f8dec790825cb379a16d7c251b1dec5e (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-13T23:38:30.537 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:30.209+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:30.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:30.354+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:30.866 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:30.574+0000 7f212b465640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:00.324693+0000 front 2024-09-13T23:38:00.324746+0000 (oldest deadline 2024-09-13T23:38:26.224644+0000) 2024-09-13T23:38:30.925 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T23:38:30.969 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:38:30.969 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T23:38:30.969 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T23:38:30.969 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.4 2024-09-13T23:38:31.227 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:31 smithi076 systemd[1]: Stopping Ceph osd.4 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:38:31.228 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:30 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:30.865+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:31.537 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:31.226+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:31.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:31.377+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:31.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:31.278+0000 7f2132e57640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:38:31.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:31.278+0000 7f2132e57640 -1 osd.4 159 *** Got signal Terminated *** 2024-09-13T23:38:31.538 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:31.278+0000 7f2132e57640 -1 osd.4 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:38:31.911 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:31.567+0000 7f212b465640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:00.324693+0000 front 2024-09-13T23:38:00.324746+0000 (oldest deadline 2024-09-13T23:38:26.224644+0000) 2024-09-13T23:38:32.234 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:31 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:31.910+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:32.527 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:32.233+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:32.527 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:32.414+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:32.787 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:32.525+0000 7f212b465640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:00.324693+0000 front 2024-09-13T23:38:00.324746+0000 (oldest deadline 2024-09-13T23:38:26.224644+0000) 2024-09-13T23:38:33.206 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:32 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:32.934+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:33.537 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:33.205+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:33.538 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:33.426+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:33.975 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:33.556+0000 7f212b465640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:00.324693+0000 front 2024-09-13T23:38:00.324746+0000 (oldest deadline 2024-09-13T23:38:26.224644+0000) 2024-09-13T23:38:34.287 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:34.200+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:34.287 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:34.200+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:07.134766+0000 front 2024-09-13T23:38:07.134712+0000 (oldest deadline 2024-09-13T23:38:33.535076+0000) 2024-09-13T23:38:34.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:33.974+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:34.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:33 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:33.974+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:34.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:34.450+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:34.788 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:34.569+0000 7f212b465640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:00.324693+0000 front 2024-09-13T23:38:00.324746+0000 (oldest deadline 2024-09-13T23:38:26.224644+0000) 2024-09-13T23:38:34.788 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:34.569+0000 7f212b465640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.125373+0000 front 2024-09-13T23:38:09.125489+0000 (oldest deadline 2024-09-13T23:38:34.425406+0000) 2024-09-13T23:38:35.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:34.978+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:35.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:34.978+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:35.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:35.183+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:35.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:35.183+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:07.134766+0000 front 2024-09-13T23:38:07.134712+0000 (oldest deadline 2024-09-13T23:38:33.535076+0000) 2024-09-13T23:38:35.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:35.402+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:35.788 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:35.597+0000 7f212b465640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:00.324693+0000 front 2024-09-13T23:38:00.324746+0000 (oldest deadline 2024-09-13T23:38:26.224644+0000) 2024-09-13T23:38:35.788 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:35 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4[74488]: 2024-09-13T23:38:35.597+0000 7f212b465640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.125373+0000 front 2024-09-13T23:38:09.125489+0000 (oldest deadline 2024-09-13T23:38:34.425406+0000) 2024-09-13T23:38:36.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:36.025+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:36.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:36.025+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:36.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:36.159+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:36.288 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:36.159+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:07.134766+0000 front 2024-09-13T23:38:07.134712+0000 (oldest deadline 2024-09-13T23:38:33.535076+0000) 2024-09-13T23:38:36.788 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:36 smithi076 podman[97316]: 2024-09-13 23:38:36.581378231 +0000 UTC m=+5.410986713 container died ba143ef88b4e45c01d25623cdacea94f60ff7fbdcde9c7723a629b869343f84d (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be) 2024-09-13T23:38:36.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:36 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:36.372+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:37.126 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:37 smithi076 podman[97316]: 2024-09-13 23:38:37.028398038 +0000 UTC m=+5.858006516 container cleanup ba143ef88b4e45c01d25623cdacea94f60ff7fbdcde9c7723a629b869343f84d (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-13T23:38:37.126 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:37 smithi076 bash[97316]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4 2024-09-13T23:38:37.126 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:37.058+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:37.126 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:37.058+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:37.537 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:37 smithi076 podman[97331]: 2024-09-13 23:38:37.338702026 +0000 UTC m=+0.751510162 container remove ba143ef88b4e45c01d25623cdacea94f60ff7fbdcde9c7723a629b869343f84d (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-ecab7be) 2024-09-13T23:38:37.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:37.357+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:37.538 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:37.357+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:37.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:37.125+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:37.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:37 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:37.125+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:07.134766+0000 front 2024-09-13T23:38:07.134712+0000 (oldest deadline 2024-09-13T23:38:33.535076+0000) 2024-09-13T23:38:38.076 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:37 smithi076 podman[97422]: 2024-09-13 23:38:37.697091983 +0000 UTC m=+0.015942628 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:38:38.076 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:37 smithi076 podman[97422]: 2024-09-13 23:38:37.82272254 +0000 UTC m=+0.141573172 container create 78fc2397de6fd677611eabe3d6015d5c9e3ac658e098d4d59398180365fe66b5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-13T23:38:38.076 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:38.021+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:38.076 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:38.021+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:38.343 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:38.162+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:38.343 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:38.162+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:07.134766+0000 front 2024-09-13T23:38:07.134712+0000 (oldest deadline 2024-09-13T23:38:33.535076+0000) 2024-09-13T23:38:38.343 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:38 smithi076 podman[97422]: 2024-09-13 23:38:38.075278191 +0000 UTC m=+0.394128826 container init 78fc2397de6fd677611eabe3d6015d5c9e3ac658e098d4d59398180365fe66b5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T23:38:38.343 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:38 smithi076 podman[97422]: 2024-09-13 23:38:38.079133215 +0000 UTC m=+0.397983854 container start 78fc2397de6fd677611eabe3d6015d5c9e3ac658e098d4d59398180365fe66b5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-13T23:38:38.344 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:38 smithi076 podman[97422]: 2024-09-13 23:38:38.169013234 +0000 UTC m=+0.487863866 container attach 78fc2397de6fd677611eabe3d6015d5c9e3ac658e098d4d59398180365fe66b5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T23:38:38.344 INFO:journalctl@ceph.osd.4.smithi076.stdout:Sep 13 23:38:38 smithi076 podman[97422]: 2024-09-13 23:38:38.282277747 +0000 UTC m=+0.601128397 container died 78fc2397de6fd677611eabe3d6015d5c9e3ac658e098d4d59398180365fe66b5 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be) 2024-09-13T23:38:38.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:38.342+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:38.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:38 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:38.342+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:38.915 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T23:38:38.954 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:38:38.954 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T23:38:38.955 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T23:38:38.955 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.5 2024-09-13T23:38:39.198 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:39 smithi076 systemd[1]: Stopping Ceph osd.5 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:38:39.198 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:39.025+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:39.198 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:39.025+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:39.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:39.388+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:39.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:39.388+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:39.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:39.196+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:39.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:39.196+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:07.134766+0000 front 2024-09-13T23:38:07.134712+0000 (oldest deadline 2024-09-13T23:38:33.535076+0000) 2024-09-13T23:38:39.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:39.343+0000 7f111b085640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:38:39.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:39.343+0000 7f111b085640 -1 osd.5 159 *** Got signal Terminated *** 2024-09-13T23:38:39.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:39.343+0000 7f111b085640 -1 osd.5 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:38:40.287 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:40 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:40.199+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:40.287 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:40 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:40.199+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:07.134766+0000 front 2024-09-13T23:38:07.134712+0000 (oldest deadline 2024-09-13T23:38:33.535076+0000) 2024-09-13T23:38:40.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:39.977+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:40.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:39 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:39.977+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:40.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:40 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:40.361+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:40.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:40 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:40.361+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:41.287 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:41.227+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:41.287 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:41.227+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:07.134766+0000 front 2024-09-13T23:38:07.134712+0000 (oldest deadline 2024-09-13T23:38:33.535076+0000) 2024-09-13T23:38:41.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:41.015+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:41.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:41.015+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:41.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:41.368+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:41.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:41 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:41.368+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:42.272 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:42.012+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:42.272 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:42.012+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:42.272 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:42.012+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:15.897116+0000 front 2024-09-13T23:38:15.897142+0000 (oldest deadline 2024-09-13T23:38:41.197005+0000) 2024-09-13T23:38:42.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:42.399+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:42.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:42.399+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:42.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:42.271+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:42.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:42.271+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:07.134766+0000 front 2024-09-13T23:38:07.134712+0000 (oldest deadline 2024-09-13T23:38:33.535076+0000) 2024-09-13T23:38:42.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:42.271+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:18.836130+0000 front 2024-09-13T23:38:18.836026+0000 (oldest deadline 2024-09-13T23:38:41.636269+0000) 2024-09-13T23:38:43.260 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:42.978+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:43.260 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:42.978+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:43.260 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:42 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:42.978+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:15.897116+0000 front 2024-09-13T23:38:15.897142+0000 (oldest deadline 2024-09-13T23:38:41.197005+0000) 2024-09-13T23:38:43.537 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:43.259+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:43.537 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:43.259+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:07.134766+0000 front 2024-09-13T23:38:07.134712+0000 (oldest deadline 2024-09-13T23:38:33.535076+0000) 2024-09-13T23:38:43.538 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:43.259+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:18.836130+0000 front 2024-09-13T23:38:18.836026+0000 (oldest deadline 2024-09-13T23:38:41.636269+0000) 2024-09-13T23:38:43.538 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:43.355+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:43.538 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:43.355+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:43.538 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:43.355+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:44.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:43.950+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:44.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:43.950+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:44.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:43 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:43.950+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:15.897116+0000 front 2024-09-13T23:38:15.897142+0000 (oldest deadline 2024-09-13T23:38:41.197005+0000) 2024-09-13T23:38:44.745 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:44 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:44.306+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:03.034608+0000 front 2024-09-13T23:38:03.034386+0000 (oldest deadline 2024-09-13T23:38:27.134209+0000) 2024-09-13T23:38:44.745 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:44 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:44.306+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:07.134766+0000 front 2024-09-13T23:38:07.134712+0000 (oldest deadline 2024-09-13T23:38:33.535076+0000) 2024-09-13T23:38:44.745 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:44 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5[79448]: 2024-09-13T23:38:44.306+0000 7f1113693640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:18.836130+0000 front 2024-09-13T23:38:18.836026+0000 (oldest deadline 2024-09-13T23:38:41.636269+0000) 2024-09-13T23:38:44.746 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:44 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:44.373+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:44.746 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:44 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:44.373+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:44.746 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:44 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:44.373+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:45.037 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:44 smithi076 podman[97535]: 2024-09-13 23:38:44.744670179 +0000 UTC m=+5.564151934 container died 57c6e1a1827dddd012c2ffe2fcf4c2c7114e0fe668d81223f5c49cdc671df2de (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T23:38:45.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:44 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:44.939+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:45.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:44 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:44.939+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:45.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:44 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:44.939+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:15.897116+0000 front 2024-09-13T23:38:15.897142+0000 (oldest deadline 2024-09-13T23:38:41.197005+0000) 2024-09-13T23:38:45.412 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:45 smithi076 podman[97535]: 2024-09-13 23:38:45.210366659 +0000 UTC m=+6.029848455 container cleanup 57c6e1a1827dddd012c2ffe2fcf4c2c7114e0fe668d81223f5c49cdc671df2de (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-13T23:38:45.413 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:45 smithi076 bash[97535]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5 2024-09-13T23:38:45.787 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:45 smithi076 podman[97550]: 2024-09-13 23:38:45.511829778 +0000 UTC m=+0.759640442 container remove 57c6e1a1827dddd012c2ffe2fcf4c2c7114e0fe668d81223f5c49cdc671df2de (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T23:38:45.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:45.410+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:45.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:45.410+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:45.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:45.410+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:46.140 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:45.930+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:46.140 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:45.930+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:46.141 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:45 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:45.930+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:15.897116+0000 front 2024-09-13T23:38:15.897142+0000 (oldest deadline 2024-09-13T23:38:41.197005+0000) 2024-09-13T23:38:46.141 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:45 smithi076 podman[97635]: 2024-09-13 23:38:45.732983016 +0000 UTC m=+0.015886092 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:38:46.141 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:45 smithi076 podman[97635]: 2024-09-13 23:38:45.846008189 +0000 UTC m=+0.128911258 container create 32847282b6e779d31c7d5c24b23e6cd53735c76da1baa7e2b103dc57348a473f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2) 2024-09-13T23:38:46.141 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:46 smithi076 podman[97635]: 2024-09-13 23:38:46.056697085 +0000 UTC m=+0.339600163 container init 32847282b6e779d31c7d5c24b23e6cd53735c76da1baa7e2b103dc57348a473f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True) 2024-09-13T23:38:46.141 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:46 smithi076 podman[97635]: 2024-09-13 23:38:46.059930153 +0000 UTC m=+0.342833222 container start 32847282b6e779d31c7d5c24b23e6cd53735c76da1baa7e2b103dc57348a473f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:38:46.141 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:46 smithi076 podman[97635]: 2024-09-13 23:38:46.1399215 +0000 UTC m=+0.422824567 container attach 32847282b6e779d31c7d5c24b23e6cd53735c76da1baa7e2b103dc57348a473f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-13T23:38:46.447 INFO:journalctl@ceph.osd.5.smithi076.stdout:Sep 13 23:38:46 smithi076 podman[97635]: 2024-09-13 23:38:46.247536274 +0000 UTC m=+0.530439348 container died 32847282b6e779d31c7d5c24b23e6cd53735c76da1baa7e2b103dc57348a473f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be) 2024-09-13T23:38:46.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:46 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:46.445+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:46.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:46 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:46.445+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:46.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:46 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:46.445+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:46.926 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T23:38:46.969 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:38:46.969 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T23:38:46.969 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T23:38:46.969 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.6 2024-09-13T23:38:47.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:46 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:46.971+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:47.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:46 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:46.971+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:47.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:46 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:46.971+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:15.897116+0000 front 2024-09-13T23:38:15.897142+0000 (oldest deadline 2024-09-13T23:38:41.197005+0000) 2024-09-13T23:38:47.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:47 smithi076 systemd[1]: Stopping Ceph osd.6 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:38:47.787 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:47.321+0000 7fd135fb2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:38:47.788 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:47.321+0000 7fd135fb2640 -1 osd.6 159 *** Got signal Terminated *** 2024-09-13T23:38:47.788 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:47.321+0000 7fd135fb2640 -1 osd.6 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:38:47.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:47.474+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:47.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:47.474+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:47.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:47.474+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:48.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:47.960+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:48.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:47.960+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:48.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:47 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:47.960+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:15.897116+0000 front 2024-09-13T23:38:15.897142+0000 (oldest deadline 2024-09-13T23:38:41.197005+0000) 2024-09-13T23:38:48.771 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:48.474+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:48.772 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:48.474+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:48.772 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:48.474+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:49.037 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:48.938+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:49.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:48.938+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:49.038 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:48 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:48.938+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:15.897116+0000 front 2024-09-13T23:38:15.897142+0000 (oldest deadline 2024-09-13T23:38:41.197005+0000) 2024-09-13T23:38:49.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:49.445+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:49.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:49.445+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:49.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:49.445+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:50.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:49.961+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:50.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:49.961+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:50.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:49.961+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:15.897116+0000 front 2024-09-13T23:38:15.897142+0000 (oldest deadline 2024-09-13T23:38:41.197005+0000) 2024-09-13T23:38:50.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:49 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:49.961+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:28.098680+0000 front 2024-09-13T23:38:28.098735+0000 (oldest deadline 2024-09-13T23:38:49.798586+0000) 2024-09-13T23:38:50.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:38:50.003Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-13T23:38:50.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:38:50.006Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-13T23:38:50.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:38:50.006Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-13T23:38:50.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:38:50.008Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-13T23:38:50.288 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:38:50.008Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-13T23:38:50.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:50.464+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:50.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:50.464+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:50.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:50.464+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:50.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:50.464+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:27.697781+0000 front 2024-09-13T23:38:27.697729+0000 (oldest deadline 2024-09-13T23:38:49.997538+0000) 2024-09-13T23:38:51.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:50.955+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:51.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:50.955+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:51.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:50.955+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:15.897116+0000 front 2024-09-13T23:38:15.897142+0000 (oldest deadline 2024-09-13T23:38:41.197005+0000) 2024-09-13T23:38:51.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:50 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:50.955+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:28.098680+0000 front 2024-09-13T23:38:28.098735+0000 (oldest deadline 2024-09-13T23:38:49.798586+0000) 2024-09-13T23:38:51.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:51.472+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:51.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:51.472+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:51.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:51.472+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:51.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:51.472+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:27.697781+0000 front 2024-09-13T23:38:27.697729+0000 (oldest deadline 2024-09-13T23:38:49.997538+0000) 2024-09-13T23:38:52.287 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:51.926+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.995364+0000 front 2024-09-13T23:38:04.995491+0000 (oldest deadline 2024-09-13T23:38:29.095192+0000) 2024-09-13T23:38:52.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:51.926+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:09.095812+0000 front 2024-09-13T23:38:09.095777+0000 (oldest deadline 2024-09-13T23:38:33.795560+0000) 2024-09-13T23:38:52.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:51.926+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:15.897116+0000 front 2024-09-13T23:38:15.897142+0000 (oldest deadline 2024-09-13T23:38:41.197005+0000) 2024-09-13T23:38:52.288 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:51 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6[84230]: 2024-09-13T23:38:51.926+0000 7fd1325c8640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:28.098680+0000 front 2024-09-13T23:38:28.098735+0000 (oldest deadline 2024-09-13T23:38:49.798586+0000) 2024-09-13T23:38:52.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:52 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:52.471+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:52.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:52 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:52.471+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:52.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:52 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:52.471+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:52.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:52 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:52.471+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:27.697781+0000 front 2024-09-13T23:38:27.697729+0000 (oldest deadline 2024-09-13T23:38:49.997538+0000) 2024-09-13T23:38:53.057 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:52 smithi076 podman[97737]: 2024-09-13 23:38:52.794147401 +0000 UTC m=+5.597018967 container died c0ab477f2b96c7a85e7a93983521ebdc1fad63611cff0bcc34dfd399289ed7ee (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True) 2024-09-13T23:38:53.407 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:53 smithi076 podman[97737]: 2024-09-13 23:38:53.238575005 +0000 UTC m=+6.041446567 container cleanup c0ab477f2b96c7a85e7a93983521ebdc1fad63611cff0bcc34dfd399289ed7ee (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-13T23:38:53.408 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:53 smithi076 bash[97737]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6 2024-09-13T23:38:53.787 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:53 smithi076 podman[97753]: 2024-09-13 23:38:53.40672456 +0000 UTC m=+0.605934329 container remove c0ab477f2b96c7a85e7a93983521ebdc1fad63611cff0bcc34dfd399289ed7ee (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:38:53.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:53 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:53.471+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:53.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:53 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:53.471+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:53.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:53 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:53.471+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:53.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:53 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:53.471+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:27.697781+0000 front 2024-09-13T23:38:27.697729+0000 (oldest deadline 2024-09-13T23:38:49.997538+0000) 2024-09-13T23:38:54.161 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:53 smithi076 podman[97845]: 2024-09-13 23:38:53.75596001 +0000 UTC m=+0.016256368 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:38:54.161 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:53 smithi076 podman[97845]: 2024-09-13 23:38:53.882737945 +0000 UTC m=+0.143034339 container create 03fa9d3d94cce9b740a562a1b277778d2fc46f0ee50561d920cc48f7fcd5d924 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-13T23:38:54.161 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:54 smithi076 podman[97845]: 2024-09-13 23:38:54.077024016 +0000 UTC m=+0.337320365 container init 03fa9d3d94cce9b740a562a1b277778d2fc46f0ee50561d920cc48f7fcd5d924 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:38:54.161 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:54 smithi076 podman[97845]: 2024-09-13 23:38:54.080311594 +0000 UTC m=+0.340607944 container start 03fa9d3d94cce9b740a562a1b277778d2fc46f0ee50561d920cc48f7fcd5d924 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-13T23:38:54.161 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:54 smithi076 podman[97845]: 2024-09-13 23:38:54.160184695 +0000 UTC m=+0.420481052 container attach 03fa9d3d94cce9b740a562a1b277778d2fc46f0ee50561d920cc48f7fcd5d924 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-ecab7be, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T23:38:54.467 INFO:journalctl@ceph.osd.6.smithi076.stdout:Sep 13 23:38:54 smithi076 podman[97845]: 2024-09-13 23:38:54.273625334 +0000 UTC m=+0.533921684 container died 03fa9d3d94cce9b740a562a1b277778d2fc46f0ee50561d920cc48f7fcd5d924 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-13T23:38:54.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:54 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:54.464+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:54.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:54 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:54.464+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:54.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:54 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:54.464+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:54.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:54 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:54.464+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:27.697781+0000 front 2024-09-13T23:38:27.697729+0000 (oldest deadline 2024-09-13T23:38:49.997538+0000) 2024-09-13T23:38:54.877 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T23:38:54.916 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:38:54.917 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T23:38:54.917 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T23:38:54.917 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.7 2024-09-13T23:38:55.251 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:54 smithi076 systemd[1]: Stopping Ceph osd.7 for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:38:55.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:55.249+0000 7fe9eeccd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T23:38:55.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:55.249+0000 7fe9eeccd640 -1 osd.7 159 *** Got signal Terminated *** 2024-09-13T23:38:55.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:55.249+0000 7fe9eeccd640 -1 osd.7 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T23:38:55.537 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:55.481+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:55.538 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:55.481+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:55.538 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:55.481+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:55.538 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:55 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:55.481+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:27.697781+0000 front 2024-09-13T23:38:27.697729+0000 (oldest deadline 2024-09-13T23:38:49.997538+0000) 2024-09-13T23:38:56.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:56.447+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:56.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:56.447+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:56.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:56.447+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:56.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:56 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:56.447+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:27.697781+0000 front 2024-09-13T23:38:27.697729+0000 (oldest deadline 2024-09-13T23:38:49.997538+0000) 2024-09-13T23:38:57.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:57.400+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:57.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:57.400+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:57.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:57.400+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:57.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:57 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:57.400+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:27.697781+0000 front 2024-09-13T23:38:27.697729+0000 (oldest deadline 2024-09-13T23:38:49.997538+0000) 2024-09-13T23:38:58.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:58.415+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:58.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:58.415+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:58.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:58.415+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:58.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:58 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:58.415+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:27.697781+0000 front 2024-09-13T23:38:27.697729+0000 (oldest deadline 2024-09-13T23:38:49.997538+0000) 2024-09-13T23:38:59.787 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:59.406+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-13T23:38:04.494172+0000 front 2024-09-13T23:38:04.494148+0000 (oldest deadline 2024-09-13T23:38:25.593995+0000) 2024-09-13T23:38:59.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:59.406+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-13T23:38:11.895942+0000 front 2024-09-13T23:38:11.895911+0000 (oldest deadline 2024-09-13T23:38:37.195740+0000) 2024-09-13T23:38:59.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:59.406+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-13T23:38:20.096902+0000 front 2024-09-13T23:38:20.096864+0000 (oldest deadline 2024-09-13T23:38:42.996702+0000) 2024-09-13T23:38:59.788 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:38:59 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7[88716]: 2024-09-13T23:38:59.406+0000 7fe9e72db640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-13T23:38:27.697781+0000 front 2024-09-13T23:38:27.697729+0000 (oldest deadline 2024-09-13T23:38:49.997538+0000) 2024-09-13T23:39:00.993 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:39:00 smithi076 podman[97949]: 2024-09-13 23:39:00.714541258 +0000 UTC m=+5.577403871 container died 4325dc4e4077936bcebbad7c5d10ebcc82a6bb91c84948eb086105f63242dc20 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T23:39:01.287 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:39:01 smithi076 podman[97949]: 2024-09-13 23:39:01.091367639 +0000 UTC m=+5.954230243 container cleanup 4325dc4e4077936bcebbad7c5d10ebcc82a6bb91c84948eb086105f63242dc20 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T23:39:01.288 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:39:01 smithi076 bash[97949]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7 2024-09-13T23:39:01.709 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:39:01 smithi076 podman[97964]: 2024-09-13 23:39:01.351165903 +0000 UTC m=+0.629433580 container remove 4325dc4e4077936bcebbad7c5d10ebcc82a6bb91c84948eb086105f63242dc20 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, ceph=True) 2024-09-13T23:39:01.972 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:39:01 smithi076 podman[98055]: 2024-09-13 23:39:01.611501933 +0000 UTC m=+0.016199716 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T23:39:01.972 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:39:01 smithi076 podman[98055]: 2024-09-13 23:39:01.729428597 +0000 UTC m=+0.134126369 container create 548182bf00393ce95069a81f29008b6c178ce782bdfcb22b273cc4e052398c8b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T23:39:01.972 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:39:01 smithi076 podman[98055]: 2024-09-13 23:39:01.971078983 +0000 UTC m=+0.375776758 container init 548182bf00393ce95069a81f29008b6c178ce782bdfcb22b273cc4e052398c8b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T23:39:02.288 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:39:01 smithi076 podman[98055]: 2024-09-13 23:39:01.974528342 +0000 UTC m=+0.379226116 container start 548182bf00393ce95069a81f29008b6c178ce782bdfcb22b273cc4e052398c8b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T23:39:02.288 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:39:02 smithi076 podman[98055]: 2024-09-13 23:39:02.054520085 +0000 UTC m=+0.459217867 container attach 548182bf00393ce95069a81f29008b6c178ce782bdfcb22b273cc4e052398c8b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T23:39:02.288 INFO:journalctl@ceph.osd.7.smithi076.stdout:Sep 13 23:39:02 smithi076 podman[98055]: 2024-09-13 23:39:02.168840134 +0000 UTC m=+0.573537908 container died 548182bf00393ce95069a81f29008b6c178ce782bdfcb22b273cc4e052398c8b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T23:39:02.790 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T23:39:02.828 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:39:02.829 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T23:39:02.829 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-13T23:39:02.829 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-9adcac8c-7223-11ef-bceb-c7b262605968@prometheus.a 2024-09-13T23:39:03.037 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:02 smithi076 systemd[1]: Stopping Ceph prometheus.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:39:03.453 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:39:03.152Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T23:39:03.453 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:39:03.152Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T23:39:03.453 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:39:03.152Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T23:39:03.453 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:39:03.152Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T23:39:03.453 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:39:03.152Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T23:39:03.453 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:39:03.152Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T23:39:03.454 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:39:03.152Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T23:39:03.454 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:39:03.152Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T23:39:03.454 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:39:03.153Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T23:39:03.454 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:39:03.153Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T23:39:03.454 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:39:03.153Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T23:39:03.454 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a[59566]: ts=2024-09-13T23:39:03.153Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T23:39:03.454 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 podman[98158]: 2024-09-13 23:39:03.163283012 +0000 UTC m=+0.112117828 container died 6f8a2fca1117fdc337860919191ee0142cb9f8dd52f08418ed756c86b58a4cfd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:39:03.454 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 podman[98158]: 2024-09-13 23:39:03.452360842 +0000 UTC m=+0.401195645 container cleanup 6f8a2fca1117fdc337860919191ee0142cb9f8dd52f08418ed756c86b58a4cfd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:39:03.454 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 bash[98158]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T23:39:03.786 INFO:journalctl@ceph.prometheus.a.smithi076.stdout:Sep 13 23:39:03 smithi076 podman[98170]: 2024-09-13 23:39:03.670422592 +0000 UTC m=+0.508986705 container remove 6f8a2fca1117fdc337860919191ee0142cb9f8dd52f08418ed756c86b58a4cfd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T23:39:03.800 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9adcac8c-7223-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-13T23:39:03.833 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T23:39:03.834 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-13T23:39:03.834 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T23:39:05.677 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:39:05 smithi022 systemd[1]: Stopping Ceph node-exporter.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:39:06.030 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:39:05 smithi022 podman[125719]: 2024-09-13 23:39:05.771965152 +0000 UTC m=+0.154743832 container died 892ef92b2fceda848c9cd944e523f6a66e58e6d5da786f1a08fc573c8315b007 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T23:39:06.388 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:39:06 smithi022 podman[125719]: 2024-09-13 23:39:06.119516412 +0000 UTC m=+0.502295090 container cleanup 892ef92b2fceda848c9cd944e523f6a66e58e6d5da786f1a08fc573c8315b007 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T23:39:06.388 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:39:06 smithi022 bash[125719]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T23:39:06.677 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:39:06 smithi022 podman[125732]: 2024-09-13 23:39:06.387288911 +0000 UTC m=+0.618891809 container remove 892ef92b2fceda848c9cd944e523f6a66e58e6d5da786f1a08fc573c8315b007 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T23:39:06.677 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:39:06 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T23:39:06.677 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:39:06 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T23:39:06.677 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:39:06 smithi022 systemd[1]: Stopped Ceph node-exporter.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:39:06.677 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 13 23:39:06 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 4.618s CPU time. 2024-09-13T23:39:07.008 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:39:06 smithi022 systemd[1]: Stopping Ceph alertmanager.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:39:07.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:39:07 smithi022 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a[58129]: ts=2024-09-13T23:39:07.007Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T23:39:07.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:39:07 smithi022 podman[125840]: 2024-09-13 23:39:07.048138229 +0000 UTC m=+0.150621912 container died dbd41932f4749ad84480de7cfc5e763c449d3030b8fa80cad55fd8cbd1b1ce91 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:39:07.675 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:39:07 smithi022 podman[125840]: 2024-09-13 23:39:07.429079585 +0000 UTC m=+0.531563257 container cleanup dbd41932f4749ad84480de7cfc5e763c449d3030b8fa80cad55fd8cbd1b1ce91 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:39:07.675 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:39:07 smithi022 bash[125840]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T23:39:07.926 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:39:07 smithi022 podman[125854]: 2024-09-13 23:39:07.675073034 +0000 UTC m=+0.616830692 container remove dbd41932f4749ad84480de7cfc5e763c449d3030b8fa80cad55fd8cbd1b1ce91 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T23:39:07.927 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:39:07 smithi022 podman[125854]: 2024-09-13 23:39:07.750754268 +0000 UTC m=+0.692511927 volume remove d1d2a0487f276dc966eb46377801e2697c882b31ae4255d77953dc4d8665e9c3 2024-09-13T23:39:07.927 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:39:07 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T23:39:07.927 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:39:07 smithi022 systemd[1]: Stopped Ceph alertmanager.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:39:07.927 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 13 23:39:07 smithi022 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 2.175s CPU time. 2024-09-13T23:39:30.947 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T23:39:32.537 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:39:32 smithi076 systemd[1]: Stopping Ceph node-exporter.b for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:39:32.982 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:39:32 smithi076 podman[98567]: 2024-09-13 23:39:32.707761693 +0000 UTC m=+0.123521188 container died 1141a03b8cd726af8bf72b48b61aea7383922413717234d32138fcfc3a58e1a5 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T23:39:33.287 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:39:33 smithi076 podman[98567]: 2024-09-13 23:39:33.180623796 +0000 UTC m=+0.596383357 container cleanup 1141a03b8cd726af8bf72b48b61aea7383922413717234d32138fcfc3a58e1a5 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T23:39:33.287 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:39:33 smithi076 bash[98567]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T23:39:33.784 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:39:33 smithi076 podman[98579]: 2024-09-13 23:39:33.431757617 +0000 UTC m=+0.728013448 container remove 1141a03b8cd726af8bf72b48b61aea7383922413717234d32138fcfc3a58e1a5 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T23:39:33.784 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:39:33 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T23:39:33.784 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:39:33 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T23:39:33.785 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:39:33 smithi076 systemd[1]: Stopped Ceph node-exporter.b for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:39:33.785 INFO:journalctl@ceph.node-exporter.b.smithi076.stdout:Sep 13 23:39:33 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 4.169s CPU time. 2024-09-13T23:39:34.036 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:39:33 smithi076 systemd[1]: Stopping Ceph grafana.a for 9adcac8c-7223-11ef-bceb-c7b262605968... 2024-09-13T23:39:34.287 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:39:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=server t=2024-09-13T23:39:34.035136001Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-13T23:39:34.288 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:39:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=tracing t=2024-09-13T23:39:34.035464058Z level=info msg="Closing tracing" 2024-09-13T23:39:34.288 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:39:34 smithi076 ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a[61780]: logger=ticker t=2024-09-13T23:39:34.035501826Z level=info msg=stopped last_tick=2024-09-13T23:39:30Z 2024-09-13T23:39:34.288 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:39:34 smithi076 podman[98689]: 2024-09-13 23:39:34.045870374 +0000 UTC m=+0.121095228 container died 79498011f0b3cf88dd5be1236e6eb6c36b63fa5d8aa49895bb8f691c376f373e (image=quay.io/ceph/grafana:9.4.12, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T23:39:34.653 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:39:34 smithi076 podman[98689]: 2024-09-13 23:39:34.401401378 +0000 UTC m=+0.476626274 container cleanup 79498011f0b3cf88dd5be1236e6eb6c36b63fa5d8aa49895bb8f691c376f373e (image=quay.io/ceph/grafana:9.4.12, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T23:39:34.653 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:39:34 smithi076 bash[98689]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a 2024-09-13T23:39:34.994 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:39:34 smithi076 podman[98701]: 2024-09-13 23:39:34.652266591 +0000 UTC m=+0.608304725 container remove 79498011f0b3cf88dd5be1236e6eb6c36b63fa5d8aa49895bb8f691c376f373e (image=quay.io/ceph/grafana:9.4.12, name=ceph-9adcac8c-7223-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T23:39:34.994 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:39:34 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-13T23:39:34.994 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:39:34 smithi076 systemd[1]: Stopped Ceph grafana.a for 9adcac8c-7223-11ef-bceb-c7b262605968. 2024-09-13T23:39:34.994 INFO:journalctl@ceph.grafana.a.smithi076.stdout:Sep 13 23:39:34 smithi076 systemd[1]: ceph-9adcac8c-7223-11ef-bceb-c7b262605968@grafana.a.service: Consumed 10.858s CPU time. 2024-09-13T23:39:46.482 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T23:39:46.518 DEBUG:teuthology.orchestra.run.smithi076:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T23:39:46.545 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T23:39:46.547 DEBUG:teuthology.misc:Transferring archived files from smithi022:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904364/remote/smithi022/crash 2024-09-13T23:39:46.547 DEBUG:teuthology.orchestra.run.smithi022:> sudo tar c -f - -C /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/crash -- . 2024-09-13T23:39:46.603 INFO:teuthology.orchestra.run.smithi022.stderr:tar: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T23:39:46.603 INFO:teuthology.orchestra.run.smithi022.stderr:tar: Error is not recoverable: exiting now 2024-09-13T23:39:46.605 DEBUG:teuthology.misc:Transferring archived files from smithi076:/var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904364/remote/smithi076/crash 2024-09-13T23:39:46.606 DEBUG:teuthology.orchestra.run.smithi076:> sudo tar c -f - -C /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/crash -- . 2024-09-13T23:39:46.645 INFO:teuthology.orchestra.run.smithi076.stderr:tar: /var/lib/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T23:39:46.645 INFO:teuthology.orchestra.run.smithi076.stderr:tar: Error is not recoverable: exiting now 2024-09-13T23:39:46.646 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T23:39:46.646 DEBUG:teuthology.orchestra.run.smithi022:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_STRAY_DAEMON | egrep -v CEPHADM_FAILED_DAEMON | egrep -v CEPHADM_AGENT_DOWN | head -n 1 2024-09-13T23:39:46.678 INFO:tasks.cephadm:Compressing logs... 2024-09-13T23:39:46.679 DEBUG:teuthology.orchestra.run.smithi022:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T23:39:46.721 DEBUG:teuthology.orchestra.run.smithi076:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T23:39:46.744 INFO:teuthology.orchestra.run.smithi022.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-09-13T23:39:46.744 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-13T23:39:46.744 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/tcmu-runner.log 2024-09-13T23:39:46.744 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-13T23:39:46.747 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/tcmu-runner.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi022.snmrkm.log 2024-09-13T23:39:46.747 INFO:teuthology.orchestra.run.smithi022.stderr: 89.3% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/tcmu-runner.log.gz 2024-09-13T23:39:46.747 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T23:39:46.747 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.1.log: /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi022.snmrkm.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T23:39:46.747 INFO:teuthology.orchestra.run.smithi022.stderr: 76.3% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi022.snmrkm.log.gz 2024-09-13T23:39:46.748 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mon.c.log 2024-09-13T23:39:46.748 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T23:39:46.748 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mon.a.log 2024-09-13T23:39:46.748 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-13T23:39:46.749 INFO:teuthology.orchestra.run.smithi076.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-13T23:39:46.749 INFO:teuthology.orchestra.run.smithi076.stderr:: No such file or directory 2024-09-13T23:39:46.749 INFO:teuthology.orchestra.run.smithi076.stderr:gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T23:39:46.749 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-13T23:39:46.749 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T23:39:46.750 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T23:39:46.750 INFO:teuthology.orchestra.run.smithi076.stderr:gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-13T23:39:46.751 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mon.a.log: 93.2% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T23:39:46.751 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.audit.log: /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-13T23:39:46.751 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.6.log: /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.5.log: 88.7% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T23:39:46.753 INFO:teuthology.orchestra.run.smithi076.stderr: 91.1% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T23:39:46.757 INFO:teuthology.orchestra.run.smithi022.stderr: 94.4% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T23:39:46.758 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mgr.y.log 2024-09-13T23:39:46.758 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.log 2024-09-13T23:39:46.759 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mgr.y.log: 90.9% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T23:39:46.761 INFO:teuthology.orchestra.run.smithi076.stderr:gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.log 2024-09-13T23:39:46.761 INFO:teuthology.orchestra.run.smithi076.stderr:gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mgr.x.log 2024-09-13T23:39:46.761 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mon.b.log 2024-09-13T23:39:46.761 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi076.rnamrn.log 2024-09-13T23:39:46.761 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mon.b.log: 90.9% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T23:39:46.763 INFO:teuthology.orchestra.run.smithi076.stderr: 89.0% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T23:39:46.773 INFO:teuthology.orchestra.run.smithi076.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T23:39:46.774 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi076.rnamrn.log: 76.5% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi076.rnamrn.log.gz 2024-09-13T23:39:46.777 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-13T23:39:46.785 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.log: 94.0% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T23:39:46.785 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T23:39:46.805 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/cephadm.log: 90.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T23:39:46.836 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.5% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T23:39:46.872 INFO:teuthology.orchestra.run.smithi076.stderr: 90.0% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mgr.x.log.gz 2024-09-13T23:39:47.238 INFO:teuthology.orchestra.run.smithi022.stderr: 89.6% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mgr.y.log.gz 2024-09-13T23:39:47.521 INFO:teuthology.orchestra.run.smithi022.stderr: 92.5% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mon.c.log.gz 2024-09-13T23:39:47.591 INFO:teuthology.orchestra.run.smithi076.stderr: 92.5% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mon.b.log.gz 2024-09-13T23:39:48.108 INFO:teuthology.orchestra.run.smithi076.stderr: 93.2% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-13T23:39:48.187 INFO:teuthology.orchestra.run.smithi022.stderr: 93.3% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-13T23:39:48.555 INFO:teuthology.orchestra.run.smithi076.stderr: 93.4% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-13T23:39:48.643 INFO:teuthology.orchestra.run.smithi022.stderr: 91.0% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-mon.a.log.gz 2024-09-13T23:39:48.698 INFO:teuthology.orchestra.run.smithi076.stderr: 93.6% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-13T23:39:48.745 INFO:teuthology.orchestra.run.smithi076.stderr: 93.8% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-13T23:39:48.747 INFO:teuthology.orchestra.run.smithi076.stderr: 2024-09-13T23:39:48.747 INFO:teuthology.orchestra.run.smithi076.stderr:real 0m2.011s 2024-09-13T23:39:48.747 INFO:teuthology.orchestra.run.smithi076.stderr:user 0m7.827s 2024-09-13T23:39:48.747 INFO:teuthology.orchestra.run.smithi076.stderr:sys 0m0.307s 2024-09-13T23:39:48.797 INFO:teuthology.orchestra.run.smithi022.stderr: 93.4% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-13T23:39:48.835 INFO:teuthology.orchestra.run.smithi022.stderr: 93.4% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-13T23:39:49.479 INFO:teuthology.orchestra.run.smithi022.stderr: 93.4% -- replaced with /var/log/ceph/9adcac8c-7223-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-13T23:39:49.481 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-09-13T23:39:49.481 INFO:teuthology.orchestra.run.smithi022.stderr:real 0m2.747s 2024-09-13T23:39:49.481 INFO:teuthology.orchestra.run.smithi022.stderr:user 0m11.073s 2024-09-13T23:39:49.482 INFO:teuthology.orchestra.run.smithi022.stderr:sys 0m0.364s 2024-09-13T23:39:49.482 INFO:tasks.cephadm:Archiving logs... 2024-09-13T23:39:49.482 DEBUG:teuthology.misc:Transferring archived files from smithi022:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904364/remote/smithi022/log 2024-09-13T23:39:49.483 DEBUG:teuthology.orchestra.run.smithi022:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T23:39:50.904 DEBUG:teuthology.misc:Transferring archived files from smithi076:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904364/remote/smithi076/log 2024-09-13T23:39:50.904 DEBUG:teuthology.orchestra.run.smithi076:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T23:39:52.041 INFO:tasks.cephadm:Removing cluster... 2024-09-13T23:39:52.042 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 --force 2024-09-13T23:39:52.340 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9adcac8c-7223-11ef-bceb-c7b262605968 --force 2024-09-13T23:39:52.615 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T23:39:52.615 DEBUG:teuthology.orchestra.run.smithi022:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T23:39:52.632 DEBUG:teuthology.orchestra.run.smithi076:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T23:39:52.648 INFO:tasks.cephadm:Teardown complete 2024-09-13T23:39:52.648 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T23:39:52.658 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T23:39:52.658 DEBUG:teuthology.orchestra.run.smithi022:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T23:39:52.675 DEBUG:teuthology.orchestra.run.smithi076:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T23:39:52.688 INFO:teuthology.orchestra.run.smithi022.stderr:bash: line 1: ntpq: command not found 2024-09-13T23:39:52.692 INFO:teuthology.orchestra.run.smithi022.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T23:39:52.693 INFO:teuthology.orchestra.run.smithi022.stdout:=============================================================================== 2024-09-13T23:39:52.693 INFO:teuthology.orchestra.run.smithi022.stdout:^* hv01.front.sepia.ceph.com 3 6 377 55 +74us[ +82us] +/- 58ms 2024-09-13T23:39:52.693 INFO:teuthology.orchestra.run.smithi022.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 55 -104us[ -96us] +/- 56ms 2024-09-13T23:39:52.693 INFO:teuthology.orchestra.run.smithi022.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 54 +47us[ +47us] +/- 89ms 2024-09-13T23:39:52.694 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T23:39:52.701 INFO:teuthology.orchestra.run.smithi076.stderr:bash: line 1: ntpq: command not found 2024-09-13T23:39:52.705 INFO:teuthology.orchestra.run.smithi076.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T23:39:52.705 INFO:teuthology.orchestra.run.smithi076.stdout:=============================================================================== 2024-09-13T23:39:52.706 INFO:teuthology.orchestra.run.smithi076.stdout:^* hv01.front.sepia.ceph.com 3 6 377 57 +81us[ +85us] +/- 58ms 2024-09-13T23:39:52.706 INFO:teuthology.orchestra.run.smithi076.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 60 -113us[ -108us] +/- 56ms 2024-09-13T23:39:52.706 INFO:teuthology.orchestra.run.smithi076.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 60 +49us[ +53us] +/- 89ms 2024-09-13T23:39:52.706 INFO:teuthology.orchestra.run.smithi076.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T23:39:52.707 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T23:39:52.716 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T23:39:52.716 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T23:39:52.724 DEBUG:teuthology.orchestra.run.smithi022:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T23:39:52.826 DEBUG:teuthology.orchestra.run.smithi076:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T23:39:52.902 DEBUG:teuthology.orchestra.run.smithi022:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-13T23:39:52.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T23:39:52.932 DEBUG:teuthology.orchestra.run.smithi076:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-13T23:39:52.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T23:39:52.960 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T23:39:52.970 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T23:39:52.979 INFO:teuthology.task.internal:Duration was 2751.735690 seconds 2024-09-13T23:39:52.980 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T23:39:52.989 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T23:39:52.989 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T23:39:52.992 DEBUG:teuthology.orchestra.run.smithi076:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T23:39:53.030 INFO:teuthology.orchestra.run.smithi022.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T23:39:53.039 INFO:teuthology.orchestra.run.smithi076.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T23:39:53.464 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T23:39:53.464 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi022.front.sepia.ceph.com 2024-09-13T23:39:53.465 DEBUG:teuthology.orchestra.run.smithi022:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T23:39:53.527 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi076.front.sepia.ceph.com 2024-09-13T23:39:53.527 DEBUG:teuthology.orchestra.run.smithi076:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T23:39:53.567 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T23:39:53.567 DEBUG:teuthology.orchestra.run.smithi022:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T23:39:53.570 DEBUG:teuthology.orchestra.run.smithi076:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T23:39:53.647 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T23:39:53.648 DEBUG:teuthology.orchestra.run.smithi022:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T23:39:53.650 DEBUG:teuthology.orchestra.run.smithi076:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T23:39:54.233 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T23:39:54.243 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T23:39:54.243 DEBUG:teuthology.orchestra.run.smithi022:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T23:39:54.269 DEBUG:teuthology.orchestra.run.smithi076:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T23:39:54.295 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T23:39:54.305 DEBUG:teuthology.orchestra.run.smithi022:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T23:39:54.311 DEBUG:teuthology.orchestra.run.smithi076:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T23:39:54.332 INFO:teuthology.orchestra.run.smithi022.stdout:kernel.core_pattern = core 2024-09-13T23:39:54.359 INFO:teuthology.orchestra.run.smithi076.stdout:kernel.core_pattern = core 2024-09-13T23:39:54.381 DEBUG:teuthology.orchestra.run.smithi022:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T23:39:54.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T23:39:54.436 DEBUG:teuthology.orchestra.run.smithi076:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T23:39:54.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T23:39:54.451 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T23:39:54.461 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T23:39:54.462 DEBUG:teuthology.misc:Transferring archived files from smithi022:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904364/remote/smithi022 2024-09-13T23:39:54.462 DEBUG:teuthology.orchestra.run.smithi022:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T23:39:54.531 DEBUG:teuthology.misc:Transferring archived files from smithi076:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904364/remote/smithi076 2024-09-13T23:39:54.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T23:39:54.573 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T23:39:54.574 DEBUG:teuthology.orchestra.run.smithi022:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T23:39:54.576 DEBUG:teuthology.orchestra.run.smithi076:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T23:39:54.622 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T23:39:54.634 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T23:39:54.634 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T23:39:54.643 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T23:39:54.643 DEBUG:teuthology.orchestra.run.smithi022:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T23:39:54.646 DEBUG:teuthology.orchestra.run.smithi076:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T23:39:54.659 INFO:teuthology.orchestra.run.smithi022.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 23:39 /home/ubuntu/cephtest 2024-09-13T23:39:54.677 INFO:teuthology.orchestra.run.smithi076.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 23:39 /home/ubuntu/cephtest 2024-09-13T23:39:54.679 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T23:39:54.688 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T23:39:54.708 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/connectivity} duration: 2751.735690355301 owner: scheduled_teuthology@teuthology success: true 2024-09-13T23:39:54.708 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T23:39:54.806 INFO:teuthology.run:pass